[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a694c8be
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a694c8be
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a694c8be

Branch: refs/heads/yang21-cdh5.7
Commit: a694c8be28f3bb6f748034a80b3d88e83c4bc9e8
Parents: 28fb758
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 12:51:11 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a694c8be/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/a694c8be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a694c8be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[1/3] kylin git commit: remove fitler hive dependency logic in AbstractHadoopJob [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 1e04972f1 -> a694c8be2 (forced update)


remove fitler hive dependency logic in AbstractHadoopJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28fb7588
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28fb7588
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28fb7588

Branch: refs/heads/yang21-hbase1.x
Commit: 28fb7588cc0d2dea64ea0b5f8a183bd4361dd16d
Parents: 4a7e8ce
Author: Hongbin Ma 
Authored: Thu Oct 13 21:35:49 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 12:44:01 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28fb7588/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index b77b6b0..5d85093 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -200,7 +200,8 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+String filteredHive = kylinHiveDependency;
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a694c8be
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a694c8be
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a694c8be

Branch: refs/heads/yang21-hbase1.x
Commit: a694c8be28f3bb6f748034a80b3d88e83c4bc9e8
Parents: 28fb758
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 12:51:11 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a694c8be/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/a694c8be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a694c8be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

kylin git commit: remove fitler hive dependency logic in AbstractHadoopJob

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21 4a7e8ce66 -> 28fb7588c


remove fitler hive dependency logic in AbstractHadoopJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28fb7588
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28fb7588
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28fb7588

Branch: refs/heads/yang21
Commit: 28fb7588cc0d2dea64ea0b5f8a183bd4361dd16d
Parents: 4a7e8ce
Author: Hongbin Ma 
Authored: Thu Oct 13 21:35:49 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 12:44:01 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28fb7588/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index b77b6b0..5d85093 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -200,7 +200,8 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+String filteredHive = kylinHiveDependency;
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);

kylin git commit: remove fitler hive dependency logic in AbstractHadoopJob

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 6bdd4f383 -> eabbd4e16


remove fitler hive dependency logic in AbstractHadoopJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eabbd4e1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eabbd4e1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eabbd4e1

Branch: refs/heads/v1.6.0-rc1
Commit: eabbd4e16869150625af43b3cefa5e9076378f48
Parents: 6bdd4f3
Author: Hongbin Ma 
Authored: Thu Oct 13 21:35:49 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 12:46:08 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eabbd4e1/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 417d63d..72f4437 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -200,7 +200,8 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+String filteredHive = kylinHiveDependency;
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);

kylin git commit: remove fitler hive dependency logic in AbstractHadoopJob

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master f2a062575 -> dd61f58a9


remove fitler hive dependency logic in AbstractHadoopJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dd61f58a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dd61f58a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dd61f58a

Branch: refs/heads/master
Commit: dd61f58a9892d7eb863b9e3e1c381d378e1014fa
Parents: f2a0625
Author: Hongbin Ma 
Authored: Thu Oct 13 21:35:49 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 12:45:16 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dd61f58a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 417d63d..72f4437 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -200,7 +200,8 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 kylinHiveDependency = kylinHiveDependency.replace(":", ",");
 
 logger.info("Hive Dependencies Before Filtered: " + 
kylinHiveDependency);
-String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+//String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
+String filteredHive = kylinHiveDependency;
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);

[1/4] kylin git commit: minor, add tool ClasspathScanner [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 9c2fbe06e -> e06bf99c0 (forced update)


minor, add tool ClasspathScanner


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a7e8ce6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a7e8ce6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a7e8ce6

Branch: refs/heads/yang21-cdh5.7
Commit: 4a7e8ce6662cd18a4cec88c2959727a21259d868
Parents: 066a38e
Author: Yang Li 
Authored: Sun Oct 30 18:40:51 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 31 11:33:33 2016 +0800

--
 .../kylin/common/util/ClasspathScanner.java | 336 +++
 .../apache/kylin/common/util/StringUtil.java|  16 +
 2 files changed, 352 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a7e8ce6/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
new file mode 100644
index 000..6079a17
--- /dev/null
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
+
+public class ClasspathScanner {
+
+public static void main(final String[] args) {
+ClasspathScanner scanner = new ClasspathScanner();
+
+System.out.println("Finding " + Arrays.toString(args) + " in:");
+
System.out.println("");
+for (File f : scanner.rootResources) {
+System.out.println("  - " + f.getAbsolutePath());
+}
+
System.out.println("");
+
+if (args.length == 0)
+return;
+
+scanner.scan("", new ResourceVisitor() {
+public void accept(File dir, String relativeFileName) {
+check(dir.getAbsolutePath(), relativeFileName.replace('\\', 
'/'));
+}
+
+public void accept(ZipFile archive, ZipEntry zipEntry) {
+check(archive.getName(), zipEntry.getName().replace('\\', 
'/'));
+}
+
+private void check(String base, String relativePath) {
+boolean hit = false;
+for (int i = 0; i < args.length && !hit; i++)
+hit = match(args[i], relativePath);
+
+if (hit) {
+System.out.println(base + " - " + relativePath);
+}
+}
+});
+}
+
+/**
+ * Scan classpath to find resources that has a suffix in name.
+ * 
+ * This thread's context class loader is used to define the searching
+ * classpath.
+ * 
+ * @param suffix
+ *like ".jsp" for example
+ * @return a string array; each element is a standard resource name 
relative
+ * to the root of class path, like "young/web/frame.jsp"
+ */
+public static String[] findResources(final String suffix) {
+ClasspathScanner scanner = new ClasspathScanner();
+
+final ArrayList result = new ArrayList();
+
+scanner.scan(suffix, new ResourceVisitor() {
+public void accept(File dir, String relativeFileName) {
+result.add(relativeFileName.replace('\\', '/'));
+}
+
+public void accept(ZipFile archive, ZipEntry zipEntry) {
+

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e04972f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e04972f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e04972f

Branch: refs/heads/yang21-cdh5.7
Commit: 1e04972f1767c78f49a1a581f472361b54e5b464
Parents: 4a7e8ce
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 31 11:38:20 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e04972f/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e04972f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e04972f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[1/3] kylin git commit: minor, add tool ClasspathScanner [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x d1382776e -> 1e04972f1 (forced update)


minor, add tool ClasspathScanner


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a7e8ce6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a7e8ce6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a7e8ce6

Branch: refs/heads/yang21-hbase1.x
Commit: 4a7e8ce6662cd18a4cec88c2959727a21259d868
Parents: 066a38e
Author: Yang Li 
Authored: Sun Oct 30 18:40:51 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 31 11:33:33 2016 +0800

--
 .../kylin/common/util/ClasspathScanner.java | 336 +++
 .../apache/kylin/common/util/StringUtil.java|  16 +
 2 files changed, 352 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a7e8ce6/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
new file mode 100644
index 000..6079a17
--- /dev/null
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
+
+public class ClasspathScanner {
+
+public static void main(final String[] args) {
+ClasspathScanner scanner = new ClasspathScanner();
+
+System.out.println("Finding " + Arrays.toString(args) + " in:");
+
System.out.println("");
+for (File f : scanner.rootResources) {
+System.out.println("  - " + f.getAbsolutePath());
+}
+
System.out.println("");
+
+if (args.length == 0)
+return;
+
+scanner.scan("", new ResourceVisitor() {
+public void accept(File dir, String relativeFileName) {
+check(dir.getAbsolutePath(), relativeFileName.replace('\\', 
'/'));
+}
+
+public void accept(ZipFile archive, ZipEntry zipEntry) {
+check(archive.getName(), zipEntry.getName().replace('\\', 
'/'));
+}
+
+private void check(String base, String relativePath) {
+boolean hit = false;
+for (int i = 0; i < args.length && !hit; i++)
+hit = match(args[i], relativePath);
+
+if (hit) {
+System.out.println(base + " - " + relativePath);
+}
+}
+});
+}
+
+/**
+ * Scan classpath to find resources that has a suffix in name.
+ * 
+ * This thread's context class loader is used to define the searching
+ * classpath.
+ * 
+ * @param suffix
+ *like ".jsp" for example
+ * @return a string array; each element is a standard resource name 
relative
+ * to the root of class path, like "young/web/frame.jsp"
+ */
+public static String[] findResources(final String suffix) {
+ClasspathScanner scanner = new ClasspathScanner();
+
+final ArrayList result = new ArrayList();
+
+scanner.scan(suffix, new ResourceVisitor() {
+public void accept(File dir, String relativeFileName) {
+result.add(relativeFileName.replace('\\', '/'));
+}
+
+public void accept(ZipFile archive, ZipEntry zipEntry) {
+

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e04972f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e04972f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e04972f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e04972f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e04972f

Branch: refs/heads/yang21-hbase1.x
Commit: 1e04972f1767c78f49a1a581f472361b54e5b464
Parents: 4a7e8ce
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 31 11:38:20 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e04972f/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[10/14] kylin git commit: minor, fix checkstyle

[mahongbin]

minor, fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e1e70a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e1e70a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e1e70a5

Branch: refs/heads/yang21-cdh5.7
Commit: 9e1e70a5abec932e820ae18111755e0830337c6b
Parents: 7601779
Author: Li Yang 
Authored: Fri Oct 28 14:55:13 2016 +0800
Committer: Li Yang 
Committed: Fri Oct 28 14:55:13 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/CubeStatsReader.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9e1e70a5/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index d31fe48..2dcb268 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -91,7 +91,7 @@ public class CubeStatsReader {
 reader = new SequenceFile.Reader(hadoopConf, seqInput);
 
 int percentage = 100;
-int mapperNumber = 0;;
+int mapperNumber = 0;
 double mapperOverlapRatio = 0;
 Map counterMap = Maps.newHashMap();
 

[03/14] kylin git commit: KYLIN-2125 add BeelineHiveClient

[mahongbin]

KYLIN-2125 add BeelineHiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6fc1c865
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6fc1c865
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6fc1c865

Branch: refs/heads/yang21-cdh5.7
Commit: 6fc1c865ae35a86839e226ae2ca7b3f88661c1dd
Parents: d5565f7
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 18:43:34 2016 +0800

--
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  88 
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 14 files changed, 769 insertions(+), 336 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index eefeba8..b7dd728 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
 String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
 tableName = dbTableName[0] + "." + dbTableName[1];
 TableDesc desc = 
cubeMgmtService.getMetadataManager().getTableDesc(tableName);
-if(desc == null)
+if (desc == null)
 return false;
 tableType = desc.getSourceType();
 
@@ -312,7 +313,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveDatabases() throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {
@@ -333,7 +334,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveTables(@PathVariable String database) 
throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/source-hive/pom.xml
--
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
 provided
 
 
+org.apache.hive
+hive-jdbc
+provided
+
+
 org.apache.mrunit
 mrunit
 hadoop2

http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 000..0fbc39b
--- /dev/null
+++ 

[14/14] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c2fbe06
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c2fbe06
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c2fbe06

Branch: refs/heads/yang21-cdh5.7
Commit: 9c2fbe06e06805ed9a39a47088991ac4059c565c
Parents: d138277
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Mon Oct 31 10:24:12 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c2fbe06/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/9c2fbe06/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9c2fbe06/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[11/14] kylin git commit: KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

[mahongbin]

KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/066a38e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/066a38e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/066a38e3

Branch: refs/heads/yang21-cdh5.7
Commit: 066a38e3b7b1c2345f9f12106173eccbcfbfb0e7
Parents: 9e1e70a
Author: shaofengshi 
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 28 22:09:13 2016 +0800

--
 .../apache/kylin/common/util/StringUtil.java| 13 +
 .../engine/mr/common/AbstractHadoopJob.java | 28 +++-
 2 files changed, 23 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/066a38e3/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
 return result;
 }
 
+public static void appendWithSeparator(StringBuilder src, String append) {
+if (src == null) {
+throw new IllegalArgumentException();
+}
+if (src.length() > 0 && src.toString().endsWith(",") == false) {
+src.append(",");
+}
+
+if (StringUtils.isBlank(append) == false) {
+src.append(append);
+}
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/066a38e3/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index af2ed9f..b77b6b0 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
-if (kylinDependency.length() > 0)
-kylinDependency.append(",");
-kylinDependency.append(filteredHive);
+StringUtil.appendWithSeparator(kylinDependency, filteredHive);
 } else {
 
 logger.info("No hive dependency jars set in the environment, will 
find them from jvm:");
 
 try {
 String hiveExecJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
-kylinDependency.append(hiveExecJarPath).append(",");
+
+StringUtil.appendWithSeparator(kylinDependency, 
hiveExecJarPath);
 logger.info("hive-exec jar file: " + hiveExecJarPath);
 
 String hiveHCatJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
-kylinDependency.append(hiveHCatJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveHCatJarPath);
 logger.info("hive-catalog jar file: " + hiveHCatJarPath);
 
 String hiveMetaStoreJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
-kylinDependency.append(hiveMetaStoreJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveMetaStoreJarPath);
 logger.info("hive-metastore jar file: " + 
hiveMetaStoreJarPath);
 } catch (ClassNotFoundException e) {
 logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for hive dependencies
 if 

[02/14] kylin git commit: KYLIN-2012 adjust rules for LookupTable & remove dead methods

[mahongbin]

KYLIN-2012 adjust rules for LookupTable & remove dead methods


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5565f77
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5565f77
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5565f77

Branch: refs/heads/yang21-cdh5.7
Commit: d5565f775fa0e907a5a2c0bc33c75a2ece91ae31
Parents: f56c3c4
Author: gaodayue 
Authored: Thu Oct 13 16:56:05 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 18:39:05 2016 +0800

--
 .../apache/kylin/metadata/MetadataManager.java  | 30 
 .../apache/kylin/source/hive/SchemaChecker.java | 27 ++
 2 files changed, 21 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d5565f77/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 8d77131..fd0a3c3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -461,11 +461,6 @@ public class MetadataManager {
 reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
 }
 
-public void reloadTableCache(String tableIdentity) throws IOException {
-reloadSourceTableExt(tableIdentity);
-reloadSourceTable(tableIdentity);
-}
-
 public DataModelDesc getDataModelDesc(String name) {
 return dataModelDescMap.get(name);
 }
@@ -622,29 +617,4 @@ public class MetadataManager {
 srcTableExdMap.remove(tableIdentity);
 }
 
-public String appendDBName(String table) {
-
-if (table.indexOf(".") > 0)
-return table;
-
-Map map = getAllTablesMap();
-
-int count = 0;
-String result = null;
-for (TableDesc t : map.values()) {
-if (t.getName().equalsIgnoreCase(table)) {
-result = t.getIdentity();
-count++;
-}
-}
-
-if (count == 1)
-return result;
-
-if (count > 1) {
-logger.warn("There are more than 1 table named with '" + table + 
"' in different database; The program couldn't determine, randomly pick '" + 
result + "'");
-}
-return result;
-}
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5565f77/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index 3b03551..0c5ccd0 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -134,7 +134,14 @@ public class SchemaChecker {
 }
 }
 
-private List checkAllUsedColumns(CubeInstance cube, TableDesc 
table, Map fieldsMap) {
+/**
+ * check whether all columns used in `cube` has compatible schema in 
current hive schema denoted by `fieldsMap`.
+ * @param cube cube to check, must use `table` in its model
+ * @param table kylin's table metadata
+ * @param fieldsMap current hive schema of `table`
+ * @return true if all columns used in `cube` has compatible schema with 
`fieldsMap`, false otherwise
+ */
+private List checkAllColumnsInCube(CubeInstance cube, TableDesc 
table, Map fieldsMap) {
 Set usedColumns = Sets.newHashSet();
 for (TblColRef col : cube.getAllColumns()) {
 usedColumns.add(col.getColumnDesc());
@@ -152,8 +159,15 @@ public class SchemaChecker {
 return violateColumns;
 }
 
-private boolean checkAllColumns(TableDesc table, List fields) 
{
-if (table.getColumnCount() != fields.size()) {
+/**
+ * check whether all columns in `table` are still in `fields` and have the 
same index as before.
+ *
+ * @param table kylin's table metadata
+ * @param fields current table metadata in hive
+ * @return true if only new columns are appended in hive, false otherwise
+ */
+private boolean checkAllColumnsInTableDesc(TableDesc table, 
List fields) {
+if (table.getColumnCount() > fields.size()) {
 return false;
 }
 
@@ -194,15 +208,16 @@ public class SchemaChecker {
 // if user reloads a fact table used by cube, then all used 

[08/14] kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

[mahongbin]

KYLIN-2134 dont reuse Object[] returned to calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d1db92b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d1db92b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d1db92b

Branch: refs/heads/yang21-cdh5.7
Commit: 5d1db92bea3bc05f02ca4e411100d58526a51cdf
Parents: 5e3ea3c
Author: Li Yang 
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 17:10:53 2016 +0800

--
 .../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java| 8 
 2 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d1db92b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class SegmentCubeTupleIterator implements ITupleIterator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements 
ITupleIterator {
 }
 GTRecord curRecord = gtItr.next();
 
-Preconditions.checkNotNull(cubeTupleConverter);
-
 // translate into tuple
 advMeasureFillers = cubeTupleConverter.translateResult(curRecord, 
tuple);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d1db92b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.query.enumerator;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Properties;
 
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator {
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {
-// make sure the tuple layout is correct
-//assert 
tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
-current = tuple.getAllValues();
+// give calcite a new array every time, see details in KYLIN-2134
+Object[] values = tuple.getAllValues();
+current = Arrays.copyOf(values, values.length);
 return current;
 }
 

[06/14] kylin git commit: minor, drop kylin.job.step.timeout

[mahongbin]

minor, drop kylin.job.step.timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9287117e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9287117e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9287117e

Branch: refs/heads/yang21-cdh5.7
Commit: 9287117e17bcd5d9f7c7b93bfa5b5d3b2ad7b0c3
Parents: 2d87fae
Author: Li Yang 
Authored: Thu Oct 27 15:11:20 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 15:11:20 2016 +0800

--
 build/conf/kylin_job_conf_inmem.xml   | 4 ++--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java| 4 
 .../java/org/apache/kylin/job/engine/JobEngineConfig.java | 7 ---
 3 files changed, 2 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index 73b5cc5..d6799d5 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -81,8 +81,8 @@
 
 
 mapred.task.timeout
-360
-Set task timeout to 1 hour
+720
+Set task timeout to 2 hour
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 53074fd..f824d8e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -407,10 +407,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptionalStringArray("kylin.job.admin.dls", null);
 }
 
-public long getJobStepTimeout() {
-return Long.parseLong(getOptional("kylin.job.step.timeout", 
String.valueOf(2 * 60 * 60)));
-}
-
 public double getJobCuboidSizeRatio() {
 return Double.parseDouble(getOptional("kylin.job.cuboid.size.ratio", 
"0.25"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java 
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 8ba1730..1e3eeaa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -145,13 +145,6 @@ public class JobEngineConfig {
 }
 
 /**
- * @return the jobStepTimeout
- */
-public long getJobStepTimeout() {
-return config.getJobStepTimeout();
-}
-
-/**
  * @return the asyncJobCheckInterval
  */
 public int getAsyncJobCheckInterval() {

[13/14] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d1382776
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d1382776
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d1382776

Branch: refs/heads/yang21-cdh5.7
Commit: d1382776e11e52e5a5fc62797aa0aaa9a12b4d72
Parents: 066a38e
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 10:21:52 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d1382776/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[07/14] kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

[mahongbin]

KYLIN-2134 Kylin will treat empty string as NULL by mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e3ea3c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e3ea3c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e3ea3c1

Branch: refs/heads/yang21-cdh5.7
Commit: 5e3ea3c13c52513bcfa6781674424276b93bb51d
Parents: 9287117
Author: Hongbin Ma 
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 15:26:07 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5e3ea3c1/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
 if (strValue == null)
 return null;
 
-if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string"))
+if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
 return null;
 
 // TODO use data type enum instead of string comparison

[01/14] kylin git commit: KYLIN-2078 Can't see generated SQL at Web UI [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 d205a506d -> 9c2fbe06e (forced update)


KYLIN-2078 Can't see generated SQL at Web UI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f56c3c40
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f56c3c40
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f56c3c40

Branch: refs/heads/yang21-cdh5.7
Commit: f56c3c40a8a80990c4624f5a35f61d0ef3f61f36
Parents: 242a72f
Author: shaofengshi 
Authored: Tue Oct 25 12:57:46 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 25 13:54:05 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java |  1 +
 .../java/org/apache/kylin/cube/ISegment.java| 39 
 .../cube/gridtable/SegmentGTStartAndEnd.java|  2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |  6 +++
 .../cube/model/CubeJoinedFlatTableEnrich.java   |  6 +++
 .../apache/kylin/gridtable/ScannerWorker.java   |  2 +-
 .../org/apache/kylin/job/JoinedFlatTable.java   | 18 +
 .../metadata/model/IJoinedFlatTableDesc.java|  2 +
 .../apache/kylin/metadata/model/ISegment.java   | 36 ++
 .../kylin/rest/controller/CubeController.java   | 13 +--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  2 +-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |  2 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  2 +-
 13 files changed, 75 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f56c3c40/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 79397c3..afb0d28 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -37,6 +37,7 @@ import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
+import org.apache.kylin.metadata.model.ISegment;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;

http://git-wip-us.apache.org/repos/asf/kylin/blob/f56c3c40/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
deleted file mode 100644
index 2e1f214..000
--- a/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- * http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-
-public interface ISegment {
-
-public String getName();
-
-public long getDateRangeStart();
-
-public long getDateRangeEnd();
-
-public long getSourceOffsetStart();
-
-public long getSourceOffsetEnd();
-
-public DataModelDesc getModel();
-
-public SegmentStatusEnum getStatus();
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/f56c3c40/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
index 21e01b9..b4a82d4 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
@@ -24,7 +24,7 @@ 

[12/14] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1382776/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1382776/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[05/14] kylin git commit: minor, remove kylin.dictionary.max.cardinality

[mahongbin]

minor, remove kylin.dictionary.max.cardinality


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2d87faec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2d87faec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2d87faec

Branch: refs/heads/yang21-cdh5.7
Commit: 2d87faec4d04e842ea73bb0493209b661305a4ee
Parents: b502a17
Author: Li Yang 
Authored: Thu Oct 27 14:34:40 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 14:35:43 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java |  4 
 .../org/apache/kylin/dict/DictionaryGenerator.java| 14 --
 2 files changed, 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2d87faec/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5d92aef..53074fd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -448,10 +448,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.cube.measure.customMeasureType.");
 }
 
-public int getDictionaryMaxCardinality() {
-return 
Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "3000"));
-}
-
 public int getTableSnapshotMaxMB() {
 return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", 
"300"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2d87faec/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index ba848c6..0adf40e 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
@@ -40,20 +39,10 @@ import com.google.common.base.Preconditions;
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class DictionaryGenerator {
 
-private static final int DICT_MAX_CARDINALITY = 
getDictionaryMaxCardinality();
-
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
 private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
 
-private static int getDictionaryMaxCardinality() {
-try {
-return 
KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
-} catch (Throwable e) {
-return 3000; // some test case does not have KylinConfig setup 
properly
-}
-}
-
 public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
 Preconditions.checkNotNull(dataType, "dataType cannot be null");
 
@@ -92,9 +81,6 @@ public class DictionaryGenerator {
 logger.debug("Dictionary cardinality: " + dict.getSize());
 logger.debug("Dictionary builder class: " + 
builder.getClass().getName());
 logger.debug("Dictionary class: " + dict.getClass().getName());
-if (dict instanceof TrieDictionary && dict.getSize() > 
DICT_MAX_CARDINALITY) {
-throw new IllegalArgumentException("Too high cardinality is not 
suitable for dictionary -- cardinality: " + dict.getSize());
-}
 return dict;
 }
 

[04/14] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[mahongbin]

port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b502a174
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b502a174
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b502a174

Branch: refs/heads/yang21-cdh5.7
Commit: b502a17414468652b75f84994fb371a767784a47
Parents: 6fc1c86
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 18:49:34 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  6 +--
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 38 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 8c64f91..0734f4f 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -38,8 +38,9 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -205,7 +206,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..b027f2e 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 346d278..1335ec3 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import 

[09/14] kylin git commit: KYLIN-2136 Enhance cubing algorithm selection

[mahongbin]

KYLIN-2136 Enhance cubing algorithm selection


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76017797
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76017797
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76017797

Branch: refs/heads/yang21-cdh5.7
Commit: 76017797bb65b3f3d5553b90817fe96a5b2d3b32
Parents: 5d1db92
Author: Li Yang 
Authored: Fri Oct 28 11:34:14 2016 +0800
Committer: Li Yang 
Committed: Fri Oct 28 14:40:31 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 +-
 .../kylin/engine/mr/common/CubeStatsReader.java | 13 +++-
 .../kylin/engine/mr/common/CubeStatsWriter.java | 76 
 .../kylin/engine/mr/common/CuboidStatsUtil.java | 73 ---
 .../mr/steps/FactDistinctColumnsReducer.java|  9 ++-
 .../engine/mr/steps/MergeStatisticsStep.java|  4 +-
 .../engine/mr/steps/SaveStatisticsStep.java | 17 +++--
 .../steps/FactDistinctColumnsReducerTest.java   |  4 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  4 +-
 9 files changed, 117 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/76017797/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f824d8e..784b2b1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -420,9 +420,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public double getCubeAlgorithmAutoThreshold() {
-return 
Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "8"));
+return 
Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "7"));
 }
 
+public int getCubeAlgorithmAutoMapperLimit() {
+return 
Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500"));
+}
+
 @Deprecated
 public int getCubeAggrGroupMaxSize() {
 return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", 
"12"));

http://git-wip-us.apache.org/repos/asf/kylin/blob/76017797/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 83e46e3..d31fe48 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -73,7 +73,8 @@ public class CubeStatsReader {
 
 final CubeSegment seg;
 final int samplingPercentage;
-final double mapperOverlapRatioOfFirstBuild; // only makes sense for the 
first build, is meaningless after merge
+final int mapperNumberOfFirstBuild; // becomes meaningless after merge
+final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after 
merge
 final Map cuboidRowEstimatesHLL;
 
 public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) 
throws IOException {
@@ -90,6 +91,7 @@ public class CubeStatsReader {
 reader = new SequenceFile.Reader(hadoopConf, seqInput);
 
 int percentage = 100;
+int mapperNumber = 0;;
 double mapperOverlapRatio = 0;
 Map counterMap = Maps.newHashMap();
 
@@ -100,7 +102,9 @@ public class CubeStatsReader {
 percentage = Bytes.toInt(value.getBytes());
 } else if (key.get() == -1) {
 mapperOverlapRatio = Bytes.toDouble(value.getBytes());
-} else {
+} else if (key.get() == -2) {
+mapperNumber = Bytes.toInt(value.getBytes());
+} else if (key.get() > 0) {
 HyperLogLogPlusCounter hll = new 
HyperLogLogPlusCounter(kylinConfig.getCubeStatsHLLPrecision());
 ByteArray byteArray = new ByteArray(value.getBytes());
 hll.readRegisters(byteArray.asBuffer());
@@ -110,6 +114,7 @@ public class CubeStatsReader {
 
 this.seg = cubeSegment;
 this.samplingPercentage = percentage;
+this.mapperNumberOfFirstBuild = mapperNumber;
 this.mapperOverlapRatioOfFirstBuild = mapperOverlapRatio;
 this.cuboidRowEstimatesHLL = counterMap;
 
@@ 

[10/13] kylin git commit: minor, fix checkstyle

[mahongbin]

minor, fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e1e70a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e1e70a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e1e70a5

Branch: refs/heads/yang21-hbase1.x
Commit: 9e1e70a5abec932e820ae18111755e0830337c6b
Parents: 7601779
Author: Li Yang 
Authored: Fri Oct 28 14:55:13 2016 +0800
Committer: Li Yang 
Committed: Fri Oct 28 14:55:13 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/common/CubeStatsReader.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9e1e70a5/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index d31fe48..2dcb268 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -91,7 +91,7 @@ public class CubeStatsReader {
 reader = new SequenceFile.Reader(hadoopConf, seqInput);
 
 int percentage = 100;
-int mapperNumber = 0;;
+int mapperNumber = 0;
 double mapperOverlapRatio = 0;
 Map counterMap = Maps.newHashMap();
 

[07/13] kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

[mahongbin]

KYLIN-2134 Kylin will treat empty string as NULL by mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e3ea3c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e3ea3c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e3ea3c1

Branch: refs/heads/yang21-hbase1.x
Commit: 5e3ea3c13c52513bcfa6781674424276b93bb51d
Parents: 9287117
Author: Hongbin Ma 
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 15:26:07 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5e3ea3c1/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
 if (strValue == null)
 return null;
 
-if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string"))
+if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
 return null;
 
 // TODO use data type enum instead of string comparison

[12/13] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1382776/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1382776/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[05/13] kylin git commit: minor, remove kylin.dictionary.max.cardinality

[mahongbin]

minor, remove kylin.dictionary.max.cardinality


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2d87faec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2d87faec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2d87faec

Branch: refs/heads/yang21-hbase1.x
Commit: 2d87faec4d04e842ea73bb0493209b661305a4ee
Parents: b502a17
Author: Li Yang 
Authored: Thu Oct 27 14:34:40 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 14:35:43 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java |  4 
 .../org/apache/kylin/dict/DictionaryGenerator.java| 14 --
 2 files changed, 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2d87faec/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5d92aef..53074fd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -448,10 +448,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.cube.measure.customMeasureType.");
 }
 
-public int getDictionaryMaxCardinality() {
-return 
Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "3000"));
-}
-
 public int getTableSnapshotMaxMB() {
 return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", 
"300"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2d87faec/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index ba848c6..0adf40e 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
@@ -40,20 +39,10 @@ import com.google.common.base.Preconditions;
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class DictionaryGenerator {
 
-private static final int DICT_MAX_CARDINALITY = 
getDictionaryMaxCardinality();
-
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
 private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
 
-private static int getDictionaryMaxCardinality() {
-try {
-return 
KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
-} catch (Throwable e) {
-return 3000; // some test case does not have KylinConfig setup 
properly
-}
-}
-
 public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
 Preconditions.checkNotNull(dataType, "dataType cannot be null");
 
@@ -92,9 +81,6 @@ public class DictionaryGenerator {
 logger.debug("Dictionary cardinality: " + dict.getSize());
 logger.debug("Dictionary builder class: " + 
builder.getClass().getName());
 logger.debug("Dictionary class: " + dict.getClass().getName());
-if (dict instanceof TrieDictionary && dict.getSize() > 
DICT_MAX_CARDINALITY) {
-throw new IllegalArgumentException("Too high cardinality is not 
suitable for dictionary -- cardinality: " + dict.getSize());
-}
 return dict;
 }
 

[04/13] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[mahongbin]

port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b502a174
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b502a174
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b502a174

Branch: refs/heads/yang21-hbase1.x
Commit: b502a17414468652b75f84994fb371a767784a47
Parents: 6fc1c86
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 18:49:34 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  6 +--
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 38 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 8c64f91..0734f4f 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -38,8 +38,9 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -205,7 +206,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..b027f2e 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502a174/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 346d278..1335ec3 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import 

[11/13] kylin git commit: KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

[mahongbin]

KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/066a38e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/066a38e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/066a38e3

Branch: refs/heads/yang21-hbase1.x
Commit: 066a38e3b7b1c2345f9f12106173eccbcfbfb0e7
Parents: 9e1e70a
Author: shaofengshi 
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 28 22:09:13 2016 +0800

--
 .../apache/kylin/common/util/StringUtil.java| 13 +
 .../engine/mr/common/AbstractHadoopJob.java | 28 +++-
 2 files changed, 23 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/066a38e3/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
 return result;
 }
 
+public static void appendWithSeparator(StringBuilder src, String append) {
+if (src == null) {
+throw new IllegalArgumentException();
+}
+if (src.length() > 0 && src.toString().endsWith(",") == false) {
+src.append(",");
+}
+
+if (StringUtils.isBlank(append) == false) {
+src.append(append);
+}
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/066a38e3/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index af2ed9f..b77b6b0 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
-if (kylinDependency.length() > 0)
-kylinDependency.append(",");
-kylinDependency.append(filteredHive);
+StringUtil.appendWithSeparator(kylinDependency, filteredHive);
 } else {
 
 logger.info("No hive dependency jars set in the environment, will 
find them from jvm:");
 
 try {
 String hiveExecJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
-kylinDependency.append(hiveExecJarPath).append(",");
+
+StringUtil.appendWithSeparator(kylinDependency, 
hiveExecJarPath);
 logger.info("hive-exec jar file: " + hiveExecJarPath);
 
 String hiveHCatJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
-kylinDependency.append(hiveHCatJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveHCatJarPath);
 logger.info("hive-catalog jar file: " + hiveHCatJarPath);
 
 String hiveMetaStoreJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
-kylinDependency.append(hiveMetaStoreJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveMetaStoreJarPath);
 logger.info("hive-metastore jar file: " + 
hiveMetaStoreJarPath);
 } catch (ClassNotFoundException e) {
 logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for hive dependencies
 

[02/13] kylin git commit: KYLIN-2012 adjust rules for LookupTable & remove dead methods

[mahongbin]

KYLIN-2012 adjust rules for LookupTable & remove dead methods


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5565f77
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5565f77
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5565f77

Branch: refs/heads/yang21-hbase1.x
Commit: d5565f775fa0e907a5a2c0bc33c75a2ece91ae31
Parents: f56c3c4
Author: gaodayue 
Authored: Thu Oct 13 16:56:05 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 18:39:05 2016 +0800

--
 .../apache/kylin/metadata/MetadataManager.java  | 30 
 .../apache/kylin/source/hive/SchemaChecker.java | 27 ++
 2 files changed, 21 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d5565f77/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 8d77131..fd0a3c3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -461,11 +461,6 @@ public class MetadataManager {
 reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
 }
 
-public void reloadTableCache(String tableIdentity) throws IOException {
-reloadSourceTableExt(tableIdentity);
-reloadSourceTable(tableIdentity);
-}
-
 public DataModelDesc getDataModelDesc(String name) {
 return dataModelDescMap.get(name);
 }
@@ -622,29 +617,4 @@ public class MetadataManager {
 srcTableExdMap.remove(tableIdentity);
 }
 
-public String appendDBName(String table) {
-
-if (table.indexOf(".") > 0)
-return table;
-
-Map map = getAllTablesMap();
-
-int count = 0;
-String result = null;
-for (TableDesc t : map.values()) {
-if (t.getName().equalsIgnoreCase(table)) {
-result = t.getIdentity();
-count++;
-}
-}
-
-if (count == 1)
-return result;
-
-if (count > 1) {
-logger.warn("There are more than 1 table named with '" + table + 
"' in different database; The program couldn't determine, randomly pick '" + 
result + "'");
-}
-return result;
-}
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5565f77/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index 3b03551..0c5ccd0 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -134,7 +134,14 @@ public class SchemaChecker {
 }
 }
 
-private List checkAllUsedColumns(CubeInstance cube, TableDesc 
table, Map fieldsMap) {
+/**
+ * check whether all columns used in `cube` has compatible schema in 
current hive schema denoted by `fieldsMap`.
+ * @param cube cube to check, must use `table` in its model
+ * @param table kylin's table metadata
+ * @param fieldsMap current hive schema of `table`
+ * @return true if all columns used in `cube` has compatible schema with 
`fieldsMap`, false otherwise
+ */
+private List checkAllColumnsInCube(CubeInstance cube, TableDesc 
table, Map fieldsMap) {
 Set usedColumns = Sets.newHashSet();
 for (TblColRef col : cube.getAllColumns()) {
 usedColumns.add(col.getColumnDesc());
@@ -152,8 +159,15 @@ public class SchemaChecker {
 return violateColumns;
 }
 
-private boolean checkAllColumns(TableDesc table, List fields) 
{
-if (table.getColumnCount() != fields.size()) {
+/**
+ * check whether all columns in `table` are still in `fields` and have the 
same index as before.
+ *
+ * @param table kylin's table metadata
+ * @param fields current table metadata in hive
+ * @return true if only new columns are appended in hive, false otherwise
+ */
+private boolean checkAllColumnsInTableDesc(TableDesc table, 
List fields) {
+if (table.getColumnCount() > fields.size()) {
 return false;
 }
 
@@ -194,15 +208,16 @@ public class SchemaChecker {
 // if user reloads a fact table used by cube, then all used 

[13/13] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d1382776
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d1382776
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d1382776

Branch: refs/heads/yang21-hbase1.x
Commit: d1382776e11e52e5a5fc62797aa0aaa9a12b4d72
Parents: 066a38e
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Oct 31 10:21:52 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d1382776/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[01/13] kylin git commit: KYLIN-2078 Can't see generated SQL at Web UI [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 8948ec7de -> d1382776e (forced update)


KYLIN-2078 Can't see generated SQL at Web UI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f56c3c40
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f56c3c40
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f56c3c40

Branch: refs/heads/yang21-hbase1.x
Commit: f56c3c40a8a80990c4624f5a35f61d0ef3f61f36
Parents: 242a72f
Author: shaofengshi 
Authored: Tue Oct 25 12:57:46 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 25 13:54:05 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java |  1 +
 .../java/org/apache/kylin/cube/ISegment.java| 39 
 .../cube/gridtable/SegmentGTStartAndEnd.java|  2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |  6 +++
 .../cube/model/CubeJoinedFlatTableEnrich.java   |  6 +++
 .../apache/kylin/gridtable/ScannerWorker.java   |  2 +-
 .../org/apache/kylin/job/JoinedFlatTable.java   | 18 +
 .../metadata/model/IJoinedFlatTableDesc.java|  2 +
 .../apache/kylin/metadata/model/ISegment.java   | 36 ++
 .../kylin/rest/controller/CubeController.java   | 13 +--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  2 +-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |  2 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  2 +-
 13 files changed, 75 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f56c3c40/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 79397c3..afb0d28 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -37,6 +37,7 @@ import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
+import org.apache.kylin.metadata.model.ISegment;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;

http://git-wip-us.apache.org/repos/asf/kylin/blob/f56c3c40/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
deleted file mode 100644
index 2e1f214..000
--- a/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- * http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-
-public interface ISegment {
-
-public String getName();
-
-public long getDateRangeStart();
-
-public long getDateRangeEnd();
-
-public long getSourceOffsetStart();
-
-public long getSourceOffsetEnd();
-
-public DataModelDesc getModel();
-
-public SegmentStatusEnum getStatus();
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/f56c3c40/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
index 21e01b9..b4a82d4 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
@@ -24,7 +24,7 

[08/13] kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

[mahongbin]

KYLIN-2134 dont reuse Object[] returned to calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d1db92b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d1db92b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d1db92b

Branch: refs/heads/yang21-hbase1.x
Commit: 5d1db92bea3bc05f02ca4e411100d58526a51cdf
Parents: 5e3ea3c
Author: Li Yang 
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 17:10:53 2016 +0800

--
 .../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java| 8 
 2 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d1db92b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class SegmentCubeTupleIterator implements ITupleIterator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements 
ITupleIterator {
 }
 GTRecord curRecord = gtItr.next();
 
-Preconditions.checkNotNull(cubeTupleConverter);
-
 // translate into tuple
 advMeasureFillers = cubeTupleConverter.translateResult(curRecord, 
tuple);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d1db92b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.query.enumerator;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Properties;
 
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator {
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {
-// make sure the tuple layout is correct
-//assert 
tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
-current = tuple.getAllValues();
+// give calcite a new array every time, see details in KYLIN-2134
+Object[] values = tuple.getAllValues();
+current = Arrays.copyOf(values, values.length);
 return current;
 }
 

[06/13] kylin git commit: minor, drop kylin.job.step.timeout

[mahongbin]

minor, drop kylin.job.step.timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9287117e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9287117e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9287117e

Branch: refs/heads/yang21-hbase1.x
Commit: 9287117e17bcd5d9f7c7b93bfa5b5d3b2ad7b0c3
Parents: 2d87fae
Author: Li Yang 
Authored: Thu Oct 27 15:11:20 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 15:11:20 2016 +0800

--
 build/conf/kylin_job_conf_inmem.xml   | 4 ++--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java| 4 
 .../java/org/apache/kylin/job/engine/JobEngineConfig.java | 7 ---
 3 files changed, 2 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index 73b5cc5..d6799d5 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -81,8 +81,8 @@
 
 
 mapred.task.timeout
-360
-Set task timeout to 1 hour
+720
+Set task timeout to 2 hour
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 53074fd..f824d8e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -407,10 +407,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptionalStringArray("kylin.job.admin.dls", null);
 }
 
-public long getJobStepTimeout() {
-return Long.parseLong(getOptional("kylin.job.step.timeout", 
String.valueOf(2 * 60 * 60)));
-}
-
 public double getJobCuboidSizeRatio() {
 return Double.parseDouble(getOptional("kylin.job.cuboid.size.ratio", 
"0.25"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9287117e/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java 
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 8ba1730..1e3eeaa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -145,13 +145,6 @@ public class JobEngineConfig {
 }
 
 /**
- * @return the jobStepTimeout
- */
-public long getJobStepTimeout() {
-return config.getJobStepTimeout();
-}
-
-/**
  * @return the asyncJobCheckInterval
  */
 public int getAsyncJobCheckInterval() {

[03/13] kylin git commit: KYLIN-2125 add BeelineHiveClient

[mahongbin]

KYLIN-2125 add BeelineHiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6fc1c865
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6fc1c865
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6fc1c865

Branch: refs/heads/yang21-hbase1.x
Commit: 6fc1c865ae35a86839e226ae2ca7b3f88661c1dd
Parents: d5565f7
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Oct 26 18:43:34 2016 +0800

--
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  88 
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 14 files changed, 769 insertions(+), 336 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index eefeba8..b7dd728 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
 String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
 tableName = dbTableName[0] + "." + dbTableName[1];
 TableDesc desc = 
cubeMgmtService.getMetadataManager().getTableDesc(tableName);
-if(desc == null)
+if (desc == null)
 return false;
 tableType = desc.getSourceType();
 
@@ -312,7 +313,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveDatabases() throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {
@@ -333,7 +334,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveTables(@PathVariable String database) 
throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/source-hive/pom.xml
--
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
 provided
 
 
+org.apache.hive
+hive-jdbc
+provided
+
+
 org.apache.mrunit
 mrunit
 hadoop2

http://git-wip-us.apache.org/repos/asf/kylin/blob/6fc1c865/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 000..0fbc39b
--- /dev/null
+++ 

[09/13] kylin git commit: KYLIN-2136 Enhance cubing algorithm selection

[mahongbin]

KYLIN-2136 Enhance cubing algorithm selection


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76017797
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76017797
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76017797

Branch: refs/heads/yang21-hbase1.x
Commit: 76017797bb65b3f3d5553b90817fe96a5b2d3b32
Parents: 5d1db92
Author: Li Yang 
Authored: Fri Oct 28 11:34:14 2016 +0800
Committer: Li Yang 
Committed: Fri Oct 28 14:40:31 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 +-
 .../kylin/engine/mr/common/CubeStatsReader.java | 13 +++-
 .../kylin/engine/mr/common/CubeStatsWriter.java | 76 
 .../kylin/engine/mr/common/CuboidStatsUtil.java | 73 ---
 .../mr/steps/FactDistinctColumnsReducer.java|  9 ++-
 .../engine/mr/steps/MergeStatisticsStep.java|  4 +-
 .../engine/mr/steps/SaveStatisticsStep.java | 17 +++--
 .../steps/FactDistinctColumnsReducerTest.java   |  4 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  4 +-
 9 files changed, 117 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/76017797/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f824d8e..784b2b1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -420,9 +420,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public double getCubeAlgorithmAutoThreshold() {
-return 
Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "8"));
+return 
Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "7"));
 }
 
+public int getCubeAlgorithmAutoMapperLimit() {
+return 
Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500"));
+}
+
 @Deprecated
 public int getCubeAggrGroupMaxSize() {
 return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", 
"12"));

http://git-wip-us.apache.org/repos/asf/kylin/blob/76017797/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 83e46e3..d31fe48 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -73,7 +73,8 @@ public class CubeStatsReader {
 
 final CubeSegment seg;
 final int samplingPercentage;
-final double mapperOverlapRatioOfFirstBuild; // only makes sense for the 
first build, is meaningless after merge
+final int mapperNumberOfFirstBuild; // becomes meaningless after merge
+final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after 
merge
 final Map cuboidRowEstimatesHLL;
 
 public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) 
throws IOException {
@@ -90,6 +91,7 @@ public class CubeStatsReader {
 reader = new SequenceFile.Reader(hadoopConf, seqInput);
 
 int percentage = 100;
+int mapperNumber = 0;;
 double mapperOverlapRatio = 0;
 Map counterMap = Maps.newHashMap();
 
@@ -100,7 +102,9 @@ public class CubeStatsReader {
 percentage = Bytes.toInt(value.getBytes());
 } else if (key.get() == -1) {
 mapperOverlapRatio = Bytes.toDouble(value.getBytes());
-} else {
+} else if (key.get() == -2) {
+mapperNumber = Bytes.toInt(value.getBytes());
+} else if (key.get() > 0) {
 HyperLogLogPlusCounter hll = new 
HyperLogLogPlusCounter(kylinConfig.getCubeStatsHLLPrecision());
 ByteArray byteArray = new ByteArray(value.getBytes());
 hll.readRegisters(byteArray.asBuffer());
@@ -110,6 +114,7 @@ public class CubeStatsReader {
 
 this.seg = cubeSegment;
 this.samplingPercentage = percentage;
+this.mapperNumberOfFirstBuild = mapperNumber;
 this.mapperOverlapRatioOfFirstBuild = mapperOverlapRatio;
 this.cuboidRowEstimatesHLL = counterMap;
 
@@ 

kylin git commit: KYLIN-2079 add explicit configuration knob for coprocessor timeout

[gaodayue]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 11c2c6919 -> 6bdd4f383


KYLIN-2079 add explicit configuration knob for coprocessor timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6bdd4f38
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6bdd4f38
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6bdd4f38

Branch: refs/heads/v1.6.0-rc1
Commit: 6bdd4f38335b3f4f37fd1e263f667d3daf5bfd73
Parents: 11c2c69
Author: gaodayue 
Authored: Tue Oct 11 15:11:38 2016 +0800
Committer: gaodayue 
Committed: Sun Oct 30 22:32:45 2016 +0800

--
 build/conf/kylin.properties |  4 ++
 .../apache/kylin/common/KylinConfigBase.java|  8 +--
 .../kylin/gridtable/StorageSideBehavior.java|  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  6 +-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java | 24 +++
 .../hbase/cube/v2/ExpectedSizeIterator.java | 73 ++--
 7 files changed, 60 insertions(+), 61 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6bdd4f38/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 33a4e76..24e8f50 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -137,6 +137,10 @@ kylin.query.mem.budget=3221225472
 
 kylin.query.coprocessor.mem.gb=3
 
+# the default coprocessor timeout is (hbase.rpc.timeout * 0.9) / 1000 seconds,
+# you can set it to a smaller value. 0 means use default.
+# kylin.query.coprocessor.timeout.seconds=0
+
 # Enable/disable ACL check for cube query
 kylin.query.security.enabled=true
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6bdd4f38/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f49127b..26c280b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -485,10 +485,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan.threshold", 
"1000"));
 }
 
-public float getCubeVisitTimeoutTimes() {
-return 
Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
-}
-
 public int getBadQueryStackTraceDepth() {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }
@@ -541,6 +537,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Double.parseDouble(this.getOptional("kylin.query.coprocessor.mem.gb", "3.0"));
 }
 
+public int getQueryCoprocessorTimeoutSeconds() {
+return 
Integer.parseInt(this.getOptional("kylin.query.coprocessor.timeout.seconds", 
"0"));
+}
+
 public boolean isQuerySecureEnabled() {
 return 
Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "true"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6bdd4f38/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
index b01ac3f..d87b41b 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/gridtable/StorageSideBehavior.java
@@ -26,7 +26,7 @@ public enum StorageSideBehavior {
 SCAN_FILTER, //only scan+filter used,used for profiling filter speed.  
Will not return any result
 SCAN_FILTER_AGGR, //aggregate the result.  Will return results
 SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
-SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY; // on each scan operation, delay for 
10s to simulate slow queries, for test use
+SCAN_FILTER_AGGR_CHECKMEM_WITHDELAY; // on each scan operation, delay for 
10ms to simulate slow queries, for test use
 
 public boolean filterToggledOn() {
 return this.ordinal() >= SCAN_FILTER.ordinal();

http://git-wip-us.apache.org/repos/asf/kylin/blob/6bdd4f38/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 

kylin git commit: minor

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 4a23f171e -> fdc47a869


minor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdc47a86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdc47a86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdc47a86

Branch: refs/heads/KYLIN-1971
Commit: fdc47a869f7e08230d1625c8669a39e050264df4
Parents: 4a23f17
Author: Yang Li 
Authored: Sun Oct 30 19:26:24 2016 +0800
Committer: Yang Li 
Committed: Sun Oct 30 19:26:24 2016 +0800

--
 .../main/java/org/apache/kylin/common/util/ClasspathScanner.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fdc47a86/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
index e7b88a9..0634599 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
@@ -57,7 +57,7 @@ public class ClasspathScanner {
 private void check(String base, String relativePath) {
 boolean hit = false;
 for (int i = 0; i < args.length && !hit; i++)
-hit = match(args[0], relativePath);
+hit = match(args[i], relativePath);
 
 if (hit) {
 System.out.println(base + " - " + relativePath);

kylin git commit: minor

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 daf359d38 -> 4a23f171e


minor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a23f171
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a23f171
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a23f171

Branch: refs/heads/KYLIN-1971
Commit: 4a23f171ec26b9da89c817eaf83159d74cb162e7
Parents: daf359d
Author: Yang Li 
Authored: Sun Oct 30 19:19:59 2016 +0800
Committer: Yang Li 
Committed: Sun Oct 30 19:19:59 2016 +0800

--
 .../kylin/common/util/ClasspathScanner.java | 30 ++--
 1 file changed, 21 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a23f171/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
index 6eeb53a..e7b88a9 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
@@ -25,6 +25,7 @@ import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
+import java.util.List;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipException;
 import java.util.zip.ZipFile;
@@ -41,6 +42,9 @@ public class ClasspathScanner {
 }
 
System.out.println("");
 
+if (args.length == 0)
+return;
+
 scanner.scan("", new ResourceVisitor() {
 public void accept(File dir, String relativeFileName) {
 check(dir.getAbsolutePath(), relativeFileName.replace('\\', 
'/'));
@@ -104,14 +108,23 @@ public class ClasspathScanner {
 }
 
 static File[] extractRoots(ClassLoader loader, boolean recursive) {
-ArrayList roots = new ArrayList();
-
-do {
-if (loader instanceof URLClassLoader) {
-URL[] urls = ((URLClassLoader) loader).getURLs();
-for (int i = 0; i < urls.length; i++) {
+List loaders = new ArrayList<>();
+while (loader != null) {
+loaders.add(loader);
+if (!recursive)
+break;
+loader = loader.getParent();
+}
+
+List roots = new ArrayList();
+
+// parent first
+for (int i = loaders.size() - 1; i >= 0; i--) {
+ClassLoader l = loaders.get(i);
+if (l instanceof URLClassLoader) {
+for (URL url : ((URLClassLoader) l).getURLs()) {
 // tricky: space is "%20" in URL
-File f = new File(urls[i].getFile().replace("%20", " "));
+File f = new File(url.getFile().replace("%20", " "));
 
 // some generated run script could contain empty path, 
i.e., foo::bar
 // try detect and filter them out
@@ -126,8 +139,7 @@ public class ClasspathScanner {
 roots.add(f);
 }
 }
-loader = loader.getParent();
-} while (loader != null && recursive);
+}
 
 return (File[]) roots.toArray(new File[roots.size()]);
 }

[21/26] kylin git commit: KYLIN-2136 Enhance cubing algorithm selection

[liyang]

KYLIN-2136 Enhance cubing algorithm selection


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a0cfaa1b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a0cfaa1b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a0cfaa1b

Branch: refs/heads/KYLIN-1971
Commit: a0cfaa1bcba920b98b663f19e4df8142019d5965
Parents: 5922830
Author: Li Yang 
Authored: Fri Oct 28 11:34:14 2016 +0800
Committer: Li Yang 
Committed: Fri Oct 28 14:52:05 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 +-
 .../kylin/engine/mr/common/CubeStatsReader.java | 13 +++-
 .../kylin/engine/mr/common/CubeStatsWriter.java | 76 
 .../kylin/engine/mr/common/CuboidStatsUtil.java | 73 ---
 .../mr/steps/FactDistinctColumnsReducer.java|  9 ++-
 .../engine/mr/steps/MergeStatisticsStep.java|  4 +-
 .../engine/mr/steps/SaveStatisticsStep.java | 17 +++--
 .../steps/FactDistinctColumnsReducerTest.java   |  4 +-
 8 files changed, 115 insertions(+), 87 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8744296..47f9878 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -424,9 +424,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public double getCubeAlgorithmAutoThreshold() {
-return 
Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "8"));
+return 
Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "7"));
 }
 
+public int getCubeAlgorithmAutoMapperLimit() {
+return 
Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500"));
+}
+
 @Deprecated
 public int getCubeAggrGroupMaxSize() {
 return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", 
"12"));

http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 83e46e3..2dcb268 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -73,7 +73,8 @@ public class CubeStatsReader {
 
 final CubeSegment seg;
 final int samplingPercentage;
-final double mapperOverlapRatioOfFirstBuild; // only makes sense for the 
first build, is meaningless after merge
+final int mapperNumberOfFirstBuild; // becomes meaningless after merge
+final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after 
merge
 final Map cuboidRowEstimatesHLL;
 
 public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) 
throws IOException {
@@ -90,6 +91,7 @@ public class CubeStatsReader {
 reader = new SequenceFile.Reader(hadoopConf, seqInput);
 
 int percentage = 100;
+int mapperNumber = 0;
 double mapperOverlapRatio = 0;
 Map counterMap = Maps.newHashMap();
 
@@ -100,7 +102,9 @@ public class CubeStatsReader {
 percentage = Bytes.toInt(value.getBytes());
 } else if (key.get() == -1) {
 mapperOverlapRatio = Bytes.toDouble(value.getBytes());
-} else {
+} else if (key.get() == -2) {
+mapperNumber = Bytes.toInt(value.getBytes());
+} else if (key.get() > 0) {
 HyperLogLogPlusCounter hll = new 
HyperLogLogPlusCounter(kylinConfig.getCubeStatsHLLPrecision());
 ByteArray byteArray = new ByteArray(value.getBytes());
 hll.readRegisters(byteArray.asBuffer());
@@ -110,6 +114,7 @@ public class CubeStatsReader {
 
 this.seg = cubeSegment;
 this.samplingPercentage = percentage;
+this.mapperNumberOfFirstBuild = mapperNumber;
 this.mapperOverlapRatioOfFirstBuild = mapperOverlapRatio;
 this.cuboidRowEstimatesHLL = counterMap;
 
@@ -141,6 +146,10 @@ public class CubeStatsReader {
 return 

[10/26] kylin git commit: KYLIN-2070 UI changes for allowing concurrent build/refresh/merge

[liyang]

KYLIN-2070 UI changes for allowing concurrent build/refresh/merge

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/efa945e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/efa945e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/efa945e5

Branch: refs/heads/KYLIN-1971
Commit: efa945e56b1d07648c3f0507831b769d0a018e7e
Parents: 7a793e5
Author: luguosheng <550175...@qq.com>
Authored: Fri Oct 21 11:40:19 2016 +0800
Committer: shaofengshi 
Committed: Thu Oct 27 09:10:23 2016 +0800

--
 webapp/app/js/controllers/cubes.js | 40 -
 webapp/app/js/services/cubes.js|  1 +
 2 files changed, 36 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/efa945e5/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index 499e93c..f735fb8 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -334,12 +334,42 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, 
$routeParams, $location,
   $scope.loadDetail(cube);
   // for streaming cube build tip
   if(cube.streaming){
-$modal.open({
-  templateUrl: 'streamingBuild.html',
-  controller: streamingBuildCtrl,
-  resolve: {
+
+SweetAlert.swal({
+  title: '',
+  text: "Are you sure to start the build?",
+  type: '',
+  showCancelButton: true,
+  confirmButtonColor: '#DD6B55',
+  confirmButtonText: "Yes",
+  closeOnConfirm: true
+}, function(isConfirm) {
+  if(isConfirm){
+loadingRequest.show();
+CubeService.rebuildStreamingCube(
+  {
+cubeId: cube.name
+  },
+  {
+sourceOffsetStart:0,
+sourceOffsetEnd:'9223372036854775807',
+buildType:'BUILD'
+  }, function (job) {
+loadingRequest.hide();
+SweetAlert.swal('Success!', 'Rebuild job was submitted 
successfully', 'success');
+  },function(e){
+
+loadingRequest.hide();
+if(e.data&& e.data.exception){
+  var message =e.data.exception;
+  var msg = !!(message) ? message : 'Failed to take action.';
+  SweetAlert.swal('Oops...', msg, 'error');
+}else{
+  SweetAlert.swal('Oops...', "Failed to take action.", 
'error');
+}
+});
   }
-});
+})
 return;
   }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/efa945e5/webapp/app/js/services/cubes.js
--
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 953f157..de60f55 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -26,6 +26,7 @@ KylinApp.factory('CubeService', ['$resource', function 
($resource, config) {
 cost: {method: 'PUT', params: {action: 'cost'}, isArray: false},
 rebuildLookUp: {method: 'PUT', params: {propName: 'segs', action: 
'refresh_lookup'}, isArray: false},
 rebuildCube: {method: 'PUT', params: {action: 'rebuild'}, isArray: false},
+rebuildStreamingCube: {method: 'PUT', params: {action: 'build2'}, isArray: 
false},
 disable: {method: 'PUT', params: {action: 'disable'}, isArray: false},
 enable: {method: 'PUT', params: {action: 'enable'}, isArray: false},
 purge: {method: 'PUT', params: {action: 'purge'}, isArray: false},

[09/26] kylin git commit: KYLIN-1726 update sampleProducer and kylin.sh

[liyang]

KYLIN-1726 update sampleProducer and kylin.sh

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a793e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a793e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a793e5c

Branch: refs/heads/KYLIN-1971
Commit: 7a793e5c33f1f7bffd5a53d64ec92065abd5856a
Parents: c804dc8
Author: shaofengshi 
Authored: Tue Oct 25 18:19:40 2016 +0800
Committer: shaofengshi 
Committed: Thu Oct 27 09:10:16 2016 +0800

--
 build/bin/kylin.sh  |  7 +++--
 .../source/kafka/util/KafkaSampleProducer.java  | 29 
 2 files changed, 22 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7a793e5c/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 9286055e..ad3a952 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -32,7 +32,6 @@ function retrieveDependency() {
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh
 source ${dir}/find-hbase-dependency.sh
-#source ${dir}/find-kafka-dependency.sh
 
 #retrive $KYLIN_EXTRA_START_OPTS
 if [ -f "${dir}/setenv.sh" ]
@@ -41,7 +40,11 @@ function retrieveDependency() {
 
 export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
 export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
-#export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
+if [ -n "$KAFKA_HOME" ]
+then
+source ${dir}/find-kafka-dependency.sh
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
+fi
 }
 
 # start command

http://git-wip-us.apache.org/repos/asf/kylin/blob/7a793e5c/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
index 3d26d3d..b8f98aa 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Random;
+import java.util.UUID;
 
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
@@ -48,7 +49,6 @@ public class KafkaSampleProducer {
 @SuppressWarnings("static-access")
 private static final Option OPTION_TOPIC = 
OptionBuilder.withArgName("topic").hasArg().isRequired(true).withDescription("Kafka
 topic").create("topic");
 private static final Option OPTION_BROKER = 
OptionBuilder.withArgName("broker").hasArg().isRequired(true).withDescription("Kafka
 broker").create("broker");
-private static final Option OPTION_DELAY = 
OptionBuilder.withArgName("delay").hasArg().isRequired(false).withDescription("Simulated
 message delay in mili-seconds, default 0").create("delay");
 private static final Option OPTION_INTERVAL = 
OptionBuilder.withArgName("interval").hasArg().isRequired(false).withDescription("Simulated
 message interval in mili-seconds, default 1000").create("interval");
 
 private static final ObjectMapper mapper = new ObjectMapper();
@@ -60,21 +60,14 @@ public class KafkaSampleProducer {
 String topic, broker;
 options.addOption(OPTION_TOPIC);
 options.addOption(OPTION_BROKER);
-options.addOption(OPTION_DELAY);
-options.addOption(OPTION_INTERVAL);
 optionsHelper.parseOptions(options, args);
 
 logger.info("options: '" + optionsHelper.getOptionsAsString() + "'");
 
 topic = optionsHelper.getOptionValue(OPTION_TOPIC);
 broker = optionsHelper.getOptionValue(OPTION_BROKER);
-long delay = 0;
-String delayString = optionsHelper.getOptionValue(OPTION_DELAY);
-if (delayString != null) {
-delay = Long.parseLong(delayString);
-}
 
-long interval = 1000;
+long interval = 10;
 String intervalString = optionsHelper.getOptionValue(OPTION_INTERVAL);
 if (intervalString != null) {
 interval = Long.parseLong(intervalString);
@@ -101,6 +94,10 @@ public class KafkaSampleProducer {
 devices.add("Andriod");
 devices.add("Other");
 
+List genders = new ArrayList();
+genders.add("Male");
+genders.add("Female");
+
 Properties props = new Properties();
 

[01/26] kylin git commit: KYLIN-1971 TblColRef != ColumnDesc, turns on [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 ab23cc242 -> daf359d38 (forced update)


KYLIN-1971 TblColRef != ColumnDesc, turns on


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea3d02ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea3d02ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea3d02ff

Branch: refs/heads/KYLIN-1971
Commit: ea3d02ff16fe96b0865b81d60063adbab3e462d2
Parents: 69213e9
Author: Li Yang 
Authored: Wed Oct 26 18:24:15 2016 +0800
Committer: Yang Li 
Committed: Thu Oct 27 08:15:44 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |   6 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  14 ++-
 .../apache/kylin/metadata/model/JoinDesc.java   |   2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |   4 +-
 .../metadata/realization/IRealization.java  |   3 +
 .../kylin/storage/hybrid/HybridInstance.java|  17 +++
 .../apache/kylin/query/ITKylinQueryTest.java|   2 +-
 .../relnode/OLAPToEnumerableConverter.java  |  53 +
 .../kylin/query/routing/ModelChooser.java   | 111 ---
 .../common/coprocessor/CoprocessorRowType.java  |   3 +
 .../v1/coprocessor/observer/RowTypeTest.java|   5 +-
 11 files changed, 146 insertions(+), 74 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2ccdde7..6bd3826 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.LookupDesc;
@@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 @Override
+public Set getAllColumnDescs() {
+return getDescriptor().listAllColumnDescs();
+}
+
+@Override
 public long getDateRangeStart() {
 List readySegs = getSegments(SegmentStatusEnum.READY);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5c73f21..0c0b4f3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
@@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 @JsonProperty("override_kylin_properties")
 private LinkedHashMap overrideKylinProps = new 
LinkedHashMap();
 
-private LinkedHashSet allColumns = new 
LinkedHashSet();
-private LinkedHashSet dimensionColumns = new 
LinkedHashSet();
+private LinkedHashSet allColumns = new LinkedHashSet<>();
+private LinkedHashSet allColumnDescs = new LinkedHashSet<>();
+private LinkedHashSet dimensionColumns = new LinkedHashSet<>();
 
 private Map derivedToHostMap = Maps.newHashMap();
 private Map hostToDerivedMap = 
Maps.newHashMap();
@@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 public Set listAllColumns() {
 return allColumns;
 }
+
+public Set listAllColumnDescs() {
+return allColumnDescs;
+}
 
 /**
  * @return dimension columns including derived, BUT NOT measures
@@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 

[17/26] kylin git commit: KYLIN-2134 Kylin will treat empty string as NULL by mistake

[liyang]

KYLIN-2134 Kylin will treat empty string as NULL by mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49260dd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49260dd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49260dd0

Branch: refs/heads/KYLIN-1971
Commit: 49260dd0cb1c129d40b51c359197402441b87256
Parents: 468fe9f
Author: Hongbin Ma 
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 15:22:57 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/49260dd0/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
 if (strValue == null)
 return null;
 
-if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string"))
+if ((strValue.equals("") || strValue.equals("\\N")) && 
!dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
 return null;
 
 // TODO use data type enum instead of string comparison

[08/26] kylin git commit: port KYLIN-2012 to new interface introduced in KYLIN-2125

[liyang]

port KYLIN-2012 to new interface introduced in KYLIN-2125


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/615e21d7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/615e21d7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/615e21d7

Branch: refs/heads/KYLIN-1971
Commit: 615e21d7fb0dae651ab643949ef5078bef7b12ea
Parents: ca6837d
Author: Hongbin Ma 
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  5 ++-
 .../kylin/source/hive/BeelineHiveClient.java|  2 +-
 .../source/hive/HiveSourceTableLoader.java  | 32 +++
 .../apache/kylin/source/hive/SchemaChecker.java | 41 
 4 files changed, 36 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4a24ad2..54feb24 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -45,8 +45,9 @@ import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
 import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.TimedJsonStreamParser;
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -234,7 +235,7 @@ public class DeployUtil {
 String tableFileDir = temp.getParent();
 temp.delete();
 
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;

http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index c8d56a5..a84aeb1 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -207,7 +207,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
 //BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test_kylin_fact_part");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 388e72b..401e720 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
@@ -34,8 +35,10 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
 import 

[22/26] kylin git commit: KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

[liyang]

KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8314e744
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8314e744
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8314e744

Branch: refs/heads/KYLIN-1971
Commit: 8314e744f40a4042f4652502c621cc30591b7fd1
Parents: a0cfaa1
Author: shaofengshi 
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 28 21:57:19 2016 +0800

--
 .../apache/kylin/common/util/StringUtil.java| 13 +
 .../engine/mr/common/AbstractHadoopJob.java | 29 ++--
 2 files changed, 22 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8314e744/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
 return result;
 }
 
+public static void appendWithSeparator(StringBuilder src, String append) {
+if (src == null) {
+throw new IllegalArgumentException();
+}
+if (src.length() > 0 && src.toString().endsWith(",") == false) {
+src.append(",");
+}
+
+if (StringUtils.isBlank(append) == false) {
+src.append(append);
+}
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8314e744/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a5b2d2e..417d63d 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 String filteredHive = 
filterKylinHiveDependency(kylinHiveDependency);
 logger.info("Hive Dependencies After Filtered: " + filteredHive);
 
-if (kylinDependency.length() > 0)
-kylinDependency.append(",");
-kylinDependency.append(filteredHive);
+StringUtil.appendWithSeparator(kylinDependency, filteredHive);
 } else {
 
 logger.info("No hive dependency jars set in the environment, will 
find them from jvm:");
 
 try {
 String hiveExecJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
-kylinDependency.append(hiveExecJarPath).append(",");
+
+StringUtil.appendWithSeparator(kylinDependency, 
hiveExecJarPath);
 logger.info("hive-exec jar file: " + hiveExecJarPath);
 
 String hiveHCatJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
-kylinDependency.append(hiveHCatJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveHCatJarPath);
 logger.info("hive-catalog jar file: " + hiveHCatJarPath);
 
 String hiveMetaStoreJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
-kylinDependency.append(hiveMetaStoreJarPath).append(",");
+StringUtil.appendWithSeparator(kylinDependency, 
hiveMetaStoreJarPath);
 logger.info("hive-metastore jar file: " + 
hiveMetaStoreJarPath);
 } catch (ClassNotFoundException e) {
 logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for kafka dependencies
 if 

[04/26] kylin git commit: KYLIN-2125 add BeelineHiveClient

[liyang]

KYLIN-2125 add BeelineHiveClient


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2a932e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2a932e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2a932e7

Branch: refs/heads/KYLIN-1971
Commit: e2a932e7439d0034b26b99897d7c293cadb1b8e3
Parents: ea3d02f
Author: Hongbin Ma 
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../kylin/rest/controller/TableController.java  |   9 +-
 source-hive/pom.xml |   5 +
 .../kylin/source/hive/BeelineHiveClient.java| 214 +++
 .../source/hive/BeelineOptionsProcessor.java|  47 
 .../apache/kylin/source/hive/CLIHiveClient.java | 169 +++
 .../apache/kylin/source/hive/HiveClient.java| 170 ---
 .../kylin/source/hive/HiveClientFactory.java|  33 +++
 .../source/hive/HiveSourceTableLoader.java  |  89 
 .../org/apache/kylin/source/hive/HiveTable.java |  16 +-
 .../apache/kylin/source/hive/HiveTableMeta.java |  71 ++
 .../kylin/source/hive/HiveTableMetaBuilder.java | 102 +
 .../apache/kylin/source/hive/HqlExecutable.java | 107 --
 .../apache/kylin/source/hive/IHiveClient.java   |  36 
 .../hive/BeelineOptionsProcessorTest.java   |  38 
 14 files changed, 769 insertions(+), 337 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d9050c1..47ff3fe 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
 import org.apache.kylin.rest.service.ModelService;
 import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
 String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
 tableName = dbTableName[0] + "." + dbTableName[1];
 TableDesc desc = 
cubeMgmtService.getMetadataManager().getTableDesc(tableName);
-if(desc == null)
+if (desc == null)
 return false;
 tableType = desc.getSourceType();
 
@@ -315,7 +316,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveDatabases() throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {
@@ -336,7 +337,7 @@ public class TableController extends BasicController {
 @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
 @ResponseBody
 private static List showHiveTables(@PathVariable String database) 
throws IOException {
-HiveClient hiveClient = new HiveClient();
+IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 List results = null;
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/pom.xml
--
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
 provided
 
 
+org.apache.hive
+hive-jdbc
+provided
+
+
 org.apache.mrunit
 mrunit
 hadoop2

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 000..0fbc39b
--- /dev/null
+++ 

[18/26] kylin git commit: KYLIN-2134 dont reuse Object[] returned to calcite

[liyang]

KYLIN-2134 dont reuse Object[] returned to calcite


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3b9be196
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3b9be196
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3b9be196

Branch: refs/heads/KYLIN-1971
Commit: 3b9be19634c8f295a9c7468bcdaf8ab2aad73fcb
Parents: 49260dd
Author: Li Yang 
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 17:09:22 2016 +0800

--
 .../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java| 8 
 2 files changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3b9be196/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Preconditions;
-
 public class SegmentCubeTupleIterator implements ITupleIterator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements 
ITupleIterator {
 }
 GTRecord curRecord = gtItr.next();
 
-Preconditions.checkNotNull(cubeTupleConverter);
-
 // translate into tuple
 advMeasureFillers = cubeTupleConverter.translateResult(curRecord, 
tuple);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3b9be196/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.query.enumerator;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Properties;
 
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator {
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {
-// make sure the tuple layout is correct
-//assert 
tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
-current = tuple.getAllValues();
+// give calcite a new array every time, see details in KYLIN-2134
+Object[] values = tuple.getAllValues();
+current = Arrays.copyOf(values, values.length);
 return current;
 }
 

[12/26] kylin git commit: Add step result check for BuildCubeWithEngine

[liyang]

Add step result check for BuildCubeWithEngine

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71963cbe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71963cbe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71963cbe

Branch: refs/heads/KYLIN-1971
Commit: 71963cbeb332a4a035d83dc18a2b090b87c15d42
Parents: 3047b53
Author: shaofengshi 
Authored: Thu Oct 27 10:48:10 2016 +0800
Committer: shaofengshi 
Committed: Thu Oct 27 10:48:10 2016 +0800

--
 .../kylin/provision/BuildCubeWithEngine.java| 119 ++-
 1 file changed, 61 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/71963cbe/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index f6c8801..3ae9bac 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -149,8 +149,7 @@ public class BuildCubeWithEngine {
 
 }
 
-
-public void after(){
+public void after() {
 DefaultScheduler.destroyInstance();
 }
 
@@ -167,11 +166,11 @@ public class BuildCubeWithEngine {
 KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(0.0f);
 }
 
-protected void waitForJob(String jobId) {
+protected ExecutableState waitForJob(String jobId) {
 while (true) {
 AbstractExecutable job = jobService.getJob(jobId);
 if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() 
== ExecutableState.ERROR) {
-break;
+return job.getStatus();
 } else {
 try {
 Thread.sleep(5000);
@@ -200,17 +199,17 @@ public class BuildCubeWithEngine {
 private void runTestAndAssertSucceed(String[] testCase) throws Exception {
 ExecutorService executorService = 
Executors.newFixedThreadPool(testCase.length);
 final CountDownLatch countDownLatch = new 
CountDownLatch(testCase.length);
-List> tasks = 
Lists.newArrayListWithExpectedSize(testCase.length);
+List tasks = 
Lists.newArrayListWithExpectedSize(testCase.length);
 for (int i = 0; i < testCase.length; i++) {
 tasks.add(executorService.submit(new TestCallable(testCase[i], 
countDownLatch)));
 }
 countDownLatch.await();
 try {
 for (int i = 0; i < tasks.size(); ++i) {
-Future task = tasks.get(i);
-final List jobIds = task.get();
-for (String jobId : jobIds) {
-assertJobSucceed(jobId);
+Future task = tasks.get(i);
+final Boolean result = task.get();
+if (result == false) {
+throw new RuntimeException("The test '" + testCase[i] + "' 
is failed.");
 }
 }
 } catch (Exception ex) {
@@ -219,13 +218,7 @@ public class BuildCubeWithEngine {
 }
 }
 
-private void assertJobSucceed(String jobId) {
-if (jobService.getOutput(jobId).getState() != ExecutableState.SUCCEED) 
{
-throw new RuntimeException("The job '" + jobId + "' is failed.");
-}
-}
-
-private class TestCallable implements Callable {
+private class TestCallable implements Callable {
 
 private final String methodName;
 private final CountDownLatch countDownLatch;
@@ -237,11 +230,11 @@ public class BuildCubeWithEngine {
 
 @SuppressWarnings("unchecked")
 @Override
-public List call() throws Exception {
+public Boolean call() throws Exception {
 try {
 final Method method = 
BuildCubeWithEngine.class.getDeclaredMethod(methodName);
 method.setAccessible(true);
-return (List) method.invoke(BuildCubeWithEngine.this);
+return (Boolean) method.invoke(BuildCubeWithEngine.this);
 } catch (Exception e) {
 logger.error(e.getMessage());
 throw e;
@@ -251,9 +244,13 @@ public class BuildCubeWithEngine {
 }
 }
 
+private void assertJobSuccess() {
+
+}
+
 @SuppressWarnings("unused")
 // called by reflection
-private List testInnerJoinCubeWithSlr() throws Exception {
+private boolean testInnerJoinCubeWithSlr() throws Exception {
 final String cubeName = "test_kylin_cube_with_slr_empty";
 clearSegment(cubeName);
 

[24/26] kylin git commit: Revert "KYLIN-2137 add check for 'kylin.job.mr.lib.dir'"

[liyang]

Revert "KYLIN-2137 add check for 'kylin.job.mr.lib.dir'"

This reverts commit c5af44353d5904ddb8e9107a160872164f0e5f4d.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/650f57c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/650f57c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/650f57c8

Branch: refs/heads/KYLIN-1971
Commit: 650f57c88c1cad890c5ca8b7679d7d94e8279080
Parents: c5af443
Author: Li Yang 
Authored: Sat Oct 29 06:48:22 2016 +0800
Committer: Li Yang 
Committed: Sat Oct 29 06:48:22 2016 +0800

--
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java| 9 +
 1 file changed, 1 insertion(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/650f57c8/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a97a8c7..417d63d 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -245,14 +245,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 
 // for KylinJobMRLibDir
 String mrLibDir = kylinConf.getKylinJobMRLibDir();
-if (!StringUtils.isBlank(mrLibDir)) {
-File dirFileMRLIB = new File(mrLibDir);
-if (dirFileMRLIB.exists()) {
-StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
-} else {
-logger.info("The directory '" + mrLibDir + "' for 
'kylin.job.mr.lib.dir' does not exist!!!");
-}
-}
+StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
 
 setJobTmpJarsAndFiles(job, kylinDependency.toString());
 

[11/26] kylin git commit: minor, remove unused entry in kylin.properties

[liyang]

minor, remove unused entry in kylin.properties


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3047b53d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3047b53d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3047b53d

Branch: refs/heads/KYLIN-1971
Commit: 3047b53d6fed4ae9a9f455223afc9dfefb758534
Parents: efa945e
Author: shaofengshi 
Authored: Wed Oct 26 11:59:44 2016 +0800
Committer: shaofengshi 
Committed: Thu Oct 27 09:10:31 2016 +0800

--
 build/conf/kylin.properties | 3 ---
 1 file changed, 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3047b53d/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..33a4e76 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -48,9 +48,6 @@ kylin.metadata.url=kylin_metadata@hbase
 # The storage for final cube file in hbase
 kylin.storage.url=hbase
 
-# In seconds (2 days)
-kylin.storage.cleanup.time.threshold=17280
-
 # Working folder in HDFS, make sure user has the right access to the hdfs 
directory
 kylin.hdfs.working.dir=/kylin
 

[15/26] kylin git commit: minor, remove kylin.dictionary.max.cardinality

[liyang]

minor, remove kylin.dictionary.max.cardinality


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a43729bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a43729bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a43729bc

Branch: refs/heads/KYLIN-1971
Commit: a43729bc75b431bbabee49e89a4e60c60cbf6baa
Parents: 5c1c24a
Author: Li Yang 
Authored: Thu Oct 27 14:34:40 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 14:34:40 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java |  4 
 .../org/apache/kylin/dict/DictionaryGenerator.java| 14 --
 2 files changed, 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a43729bc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f49127b..455b691 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -452,10 +452,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.cube.measure.customMeasureType.");
 }
 
-public int getDictionaryMaxCardinality() {
-return 
Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "3000"));
-}
-
 public int getTableSnapshotMaxMB() {
 return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", 
"300"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a43729bc/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index ba848c6..0adf40e 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
@@ -40,20 +39,10 @@ import com.google.common.base.Preconditions;
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class DictionaryGenerator {
 
-private static final int DICT_MAX_CARDINALITY = 
getDictionaryMaxCardinality();
-
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
 private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
 
-private static int getDictionaryMaxCardinality() {
-try {
-return 
KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
-} catch (Throwable e) {
-return 3000; // some test case does not have KylinConfig setup 
properly
-}
-}
-
 public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
 Preconditions.checkNotNull(dataType, "dataType cannot be null");
 
@@ -92,9 +81,6 @@ public class DictionaryGenerator {
 logger.debug("Dictionary cardinality: " + dict.getSize());
 logger.debug("Dictionary builder class: " + 
builder.getClass().getName());
 logger.debug("Dictionary class: " + dict.getClass().getName());
-if (dict instanceof TrieDictionary && dict.getSize() > 
DICT_MAX_CARDINALITY) {
-throw new IllegalArgumentException("Too high cardinality is not 
suitable for dictionary -- cardinality: " + dict.getSize());
-}
 return dict;
 }
 

[13/26] kylin git commit: KYLIN-2116 Remove field delimiter in hive flatten table ddl

[liyang]

KYLIN-2116 Remove field delimiter in hive flatten table ddl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b3047a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b3047a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b3047a7

Branch: refs/heads/KYLIN-1971
Commit: 4b3047a767aa7a696bf663cab96c11894a1f901e
Parents: 71963cb
Author: lidongsjtu 
Authored: Wed Oct 26 22:49:19 2016 +0800
Committer: lidongsjtu 
Committed: Thu Oct 27 11:15:51 2016 +0800

--
 core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3047a7/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 3cc27ba..9c73a53 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -92,8 +92,6 @@ public class JoinedFlatTable {
 ddl.append(colName(col.getCanonicalName()) + " " + 
getHiveDataType(col.getDatatype()) + "\n");
 }
 ddl.append(")" + "\n");
-
-ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'" + "\n");
 ddl.append("STORED AS SEQUENCEFILE" + "\n");
 ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + 
"';").append("\n");
 // ddl.append("TBLPROPERTIES ('serialization.null.format'='N')" +

[14/26] kylin git commit: KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as QueryId

[liyang]

KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as 
QueryId


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c1c24a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c1c24a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c1c24a9

Branch: refs/heads/KYLIN-1971
Commit: 5c1c24a9f7cc5635e0fde5eb89be595f6a07ea25
Parents: 4b3047a
Author: gaodayue 
Authored: Thu Oct 27 12:14:49 2016 +0800
Committer: gaodayue 
Committed: Thu Oct 27 12:16:36 2016 +0800

--
 .../apache/kylin/rest/service/QueryService.java |   8 +-
 .../kylin/rest/util/QueryIdGenerator.java   |  47 
 .../kylin/rest/util/QueryIdGeneratorTest.java   | 108 ---
 .../coprocessor/endpoint/CubeVisitService.java  |   2 +-
 4 files changed, 4 insertions(+), 161 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1c24a9/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 94f2dd5..81af044 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -39,6 +39,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 
 import javax.annotation.PostConstruct;
 import javax.sql.DataSource;
@@ -72,7 +73,6 @@ import org.apache.kylin.rest.model.TableMeta;
 import org.apache.kylin.rest.request.PrepareSqlRequest;
 import org.apache.kylin.rest.request.SQLRequest;
 import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.util.QueryIdGenerator;
 import org.apache.kylin.rest.util.QueryUtil;
 import org.apache.kylin.rest.util.Serializer;
 import org.apache.kylin.rest.util.TableauInterceptor;
@@ -117,8 +117,6 @@ public class QueryService extends BasicService {
 private final String hbaseUrl;
 private final String userTableName;
 
-private QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
-
 @Autowired
 private CacheManager cacheManager;
 
@@ -325,7 +323,7 @@ public class QueryService extends BasicService {
 throw new InternalErrorException("Query is not allowed in " + 
serverMode + " mode.");
 }
 
-final String queryId = 
queryIdGenerator.nextId(sqlRequest.getProject());
+final String queryId = UUID.randomUUID().toString();
 
 Map toggles = new HashMap<>();
 toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
@@ -334,7 +332,7 @@ public class QueryService extends BasicService {
 }
 BackdoorToggles.setToggles(toggles);
 
-try (SetThreadName ignored = new SetThreadName("Query-%s", queryId)) {
+try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
 String sql = sqlRequest.getSql();
 String project = sqlRequest.getProject();
 logger.info("Using project: " + project);

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1c24a9/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
deleted file mode 100644
index 2dd19c2..000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import org.apache.commons.lang3.time.FastDateFormat;
-
-import javax.annotation.Nonnull;
-import javax.annotation.concurrent.ThreadSafe;
-import 

[26/26] kylin git commit: minor, tool ClasspathScanner

[liyang]

minor, tool ClasspathScanner


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/daf359d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/daf359d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/daf359d3

Branch: refs/heads/KYLIN-1971
Commit: daf359d38e9339c9c1151e5b90a0449d05841e19
Parents: 635b3b9
Author: Yang Li 
Authored: Sun Oct 30 18:40:51 2016 +0800
Committer: Yang Li 
Committed: Sun Oct 30 18:40:51 2016 +0800

--
 .../kylin/common/util/ClasspathScanner.java | 335 +++
 .../apache/kylin/common/util/StringUtil.java|  16 +
 2 files changed, 351 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/daf359d3/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
new file mode 100644
index 000..6eeb53a
--- /dev/null
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
+
+public class ClasspathScanner {
+
+public static void main(final String[] args) {
+ClasspathScanner scanner = new ClasspathScanner();
+
+System.out.println("Finding " + Arrays.toString(args) + " in:");
+
System.out.println("");
+for (File f : scanner.rootResources) {
+System.out.println("  - " + f.getAbsolutePath());
+}
+
System.out.println("");
+
+scanner.scan("", new ResourceVisitor() {
+public void accept(File dir, String relativeFileName) {
+check(dir.getAbsolutePath(), relativeFileName.replace('\\', 
'/'));
+}
+
+public void accept(ZipFile archive, ZipEntry zipEntry) {
+check(archive.getName(), zipEntry.getName().replace('\\', 
'/'));
+}
+
+private void check(String base, String relativePath) {
+boolean hit = false;
+for (int i = 0; i < args.length && !hit; i++)
+hit = match(args[0], relativePath);
+
+if (hit) {
+System.out.println(base + " - " + relativePath);
+}
+}
+});
+}
+
+/**
+ * Scan classpath to find resources that has a suffix in name.
+ * 
+ * This thread's context class loader is used to define the searching
+ * classpath.
+ * 
+ * @param suffix
+ *like ".jsp" for example
+ * @return a string array; each element is a standard resource name 
relative
+ * to the root of class path, like "young/web/frame.jsp"
+ */
+public static String[] findResources(final String suffix) {
+ClasspathScanner scanner = new ClasspathScanner();
+
+final ArrayList result = new ArrayList();
+
+scanner.scan(suffix, new ResourceVisitor() {
+public void accept(File dir, String relativeFileName) {
+result.add(relativeFileName.replace('\\', '/'));
+}
+
+public void accept(ZipFile archive, ZipEntry zipEntry) {
+result.add(zipEntry.getName().replace('\\', '/'));
+}
+});
+
+return (String[]) result.toArray(new String[result.size()]);
+}
+
+// 

[03/26] kylin git commit: port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)

[liyang]

port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca6837d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca6837d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca6837d0

Branch: refs/heads/KYLIN-1971
Commit: ca6837d004c0da31c95027b48db589238179f1ed
Parents: e2a932e
Author: Hongbin Ma 
Authored: Wed Oct 26 13:37:03 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../org/apache/kylin/source/hive/BeelineHiveClient.java   | 10 +-
 .../java/org/apache/kylin/source/hive/CLIHiveClient.java  |  4 ++--
 .../apache/kylin/source/hive/HiveSourceTableLoader.java   |  1 +
 .../java/org/apache/kylin/source/hive/HiveTableMeta.java  |  6 --
 4 files changed, 12 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..c8d56a5 100644
--- 
a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ 
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -115,7 +115,7 @@ public class BeelineHiveClient implements IHiveClient {
 
 List allColumns = 
Lists.newArrayList();
 while (columns.next()) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), 
columns.getString(12)));
 }
 builder.setAllColumns(allColumns);
 
@@ -139,7 +139,7 @@ public class BeelineHiveClient implements IHiveClient {
 if ("".equals(resultSet.getString(1).trim())) {
 break;
 }
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim()));
+partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), 
resultSet.getString(2).trim(), resultSet.getString(3).trim()));
 }
 builder.setPartitionColumns(partitionColumns);
 }
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
 
 public static void main(String[] args) throws SQLException {
 
-//BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
-BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
-HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"events");
+BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://sandbox:1'");
+//BeelineHiveClient loader = new 
BeelineHiveClient(StringUtils.join(args, " "));
+HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", 
"test001");
 System.out.println(hiveTableMeta);
 loader.close();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index ea74470..60cf47a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -89,11 +89,11 @@ public class CLIHiveClient implements IHiveClient {
 List allColumns = 
Lists.newArrayList();
 List partitionColumns = 
Lists.newArrayList();
 for (FieldSchema fieldSchema : allFields) {
-allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 
fieldSchema.getType()));
+allColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), 
fieldSchema.getComment()));
 }
 if (partitionFields != null && partitionFields.size() > 0) {
 for (FieldSchema fieldSchema : partitionFields) {
-partitionColumns.add(new 
HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), 

[16/26] kylin git commit: minor, drop kylin.job.step.timeout

[liyang]

minor, drop kylin.job.step.timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/468fe9ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/468fe9ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/468fe9ff

Branch: refs/heads/KYLIN-1971
Commit: 468fe9ffc3d1b6c6813481803dd59673f78718f1
Parents: a43729b
Author: Li Yang 
Authored: Thu Oct 27 15:01:51 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 15:01:51 2016 +0800

--
 build/conf/kylin_job_conf_inmem.xml   | 2 +-
 .../main/java/org/apache/kylin/common/KylinConfigBase.java| 4 
 .../java/org/apache/kylin/job/engine/JobEngineConfig.java | 7 ---
 3 files changed, 1 insertion(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index 1d349f6..d363507 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -81,7 +81,7 @@
 
 
 mapreduce.task.timeout
-360
+720
 Set task timeout to 1 hour
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 455b691..8744296 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -411,10 +411,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptionalStringArray("kylin.job.admin.dls", null);
 }
 
-public long getJobStepTimeout() {
-return Long.parseLong(getOptional("kylin.job.step.timeout", 
String.valueOf(2 * 60 * 60)));
-}
-
 public double getJobCuboidSizeRatio() {
 return Double.parseDouble(getOptional("kylin.job.cuboid.size.ratio", 
"0.25"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java 
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 8ba1730..1e3eeaa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -145,13 +145,6 @@ public class JobEngineConfig {
 }
 
 /**
- * @return the jobStepTimeout
- */
-public long getJobStepTimeout() {
-return config.getJobStepTimeout();
-}
-
-/**
  * @return the asyncJobCheckInterval
  */
 public int getAsyncJobCheckInterval() {

[07/26] kylin git commit: improve integer type upgrade logic

[liyang]

improve integer type upgrade logic


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1429da5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1429da5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1429da5f

Branch: refs/heads/KYLIN-1971
Commit: 1429da5f650852402d1494785368e9175f13c379
Parents: 615e21d
Author: Hongbin Ma 
Authored: Fri Sep 30 18:31:04 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../filter/EvaluatableFunctionTupleFilter.java  | 151 ---
 .../metadata/filter/TupleFilterSerializer.java  |  20 ++-
 .../apache/kylin/metadata/model/ColumnDesc.java |  20 ++-
 .../apache/kylin/metadata/tuple/TupleInfo.java  |   4 +-
 .../apache/kylin/query/schema/OLAPTable.java|  31 +++-
 5 files changed, 63 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
deleted file mode 100644
index ff24172..000
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- * http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.lang.reflect.InvocationTargetException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-import com.google.common.collect.Lists;
-
-public class EvaluatableFunctionTupleFilter extends BuiltInFunctionTupleFilter 
{
-
-private boolean constantsInitted = false;
-
-//about non-like
-private List values;
-private Object tupleValue;
-
-public EvaluatableFunctionTupleFilter(String name) {
-super(name, FilterOperatorEnum.EVAL_FUNC);
-values = Lists.newArrayListWithCapacity(1);
-values.add(null);
-}
-
-@Override
-public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
-
-// extract tuple value
-Object tupleValue = null;
-for (TupleFilter filter : this.children) {
-if (!isConstant(filter)) {
-filter.evaluate(tuple, cs);
-tupleValue = filter.getValues().iterator().next();
-}
-}
-
-TblColRef tblColRef = this.getColumn();
-DataType strDataType = DataType.getType("string");
-if (tblColRef.getType() != strDataType) {
-throw new IllegalStateException("Only String type is allow in 
BuiltInFunction");
-}
-ByteArray valueByteArray = (ByteArray) tupleValue;
-StringSerializer serializer = new StringSerializer(strDataType);
-String value = 
serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), 
valueByteArray.offset(), valueByteArray.length()));
-
-try {
-if (isLikeFunction()) {
-return (Boolean) invokeFunction(value);
-} else {
-this.tupleValue = invokeFunction(value);
-//convert back to ByteArray format because the outer 
EvaluatableFunctionTupleFilter assumes input as ByteArray
-ByteBuffer buffer = 
ByteBuffer.allocate(valueByteArray.length() * 2);
-serializer.serialize((String) this.tupleValue, buffer);
-this.tupleValue = new ByteArray(buffer.array(), 0, 
buffer.position());
-
-

[05/26] kylin git commit: KYLIN-2030 bug fix

[liyang]

KYLIN-2030 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/324e4239
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/324e4239
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/324e4239

Branch: refs/heads/KYLIN-1971
Commit: 324e4239c308ba5b14a46eb3187dad3186489f92
Parents: 46284bd
Author: Hongbin Ma 
Authored: Sun Oct 9 19:07:52 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../kylin/cube/CubeCapabilityChecker.java   | 23 ++--
 .../kylin/query/relnode/OLAPAggregateRel.java   | 22 +++
 2 files changed, 25 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/324e4239/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index ee21b1c..e509d98 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -81,7 +81,7 @@ public class CubeCapabilityChecker {
 //1. dimension as measure
 
 if (!unmatchedAggregations.isEmpty()) {
-tryDimensionAsMeasures(unmatchedAggregations, digest, cube, 
result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
+tryDimensionAsMeasures(unmatchedAggregations, result, 
cube.getDescriptor().listDimensionColumnsIncludingDerived());
 }
 } else {
 //for non query-on-facttable 
@@ -92,10 +92,18 @@ public class CubeCapabilityChecker {
 dimCols.add(columnDesc.getRef());
 }
 
-//1. dimension as measure, like max(cal_dt) or count( distinct 
col) from lookup
+//1. all aggregations on lookup table can be done. For 
distinct count, mark them all DimensionAsMeasures
+// so that the measure has a chance to be upgraded to 
DimCountDistinctMeasureType in 
org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType
 if (!unmatchedAggregations.isEmpty()) {
-tryDimensionAsMeasures(unmatchedAggregations, digest, 
cube, result, dimCols);
+Iterator itr = 
unmatchedAggregations.iterator();
+while (itr.hasNext()) {
+FunctionDesc functionDesc = itr.next();
+if 
(dimCols.containsAll(functionDesc.getParameter().getColRefs())) {
+itr.remove();
+}
+}
 }
+tryDimensionAsMeasures(Lists.newArrayList(aggrFunctions), 
result, dimCols);
 
 //2. more "dimensions" contributed by snapshot
 if (!unmatchedDimensions.isEmpty()) {
@@ -159,19 +167,12 @@ public class CubeCapabilityChecker {
 return result;
 }
 
-private static void tryDimensionAsMeasures(Collection 
unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult 
result, Set dimCols) {
-CubeDesc cubeDesc = cube.getDescriptor();
-Collection cubeFuncs = cubeDesc.listAllFunctions();
+private static void tryDimensionAsMeasures(Collection 
unmatchedAggregations, CapabilityResult result, Set dimCols) {
 
 Iterator it = unmatchedAggregations.iterator();
 while (it.hasNext()) {
 FunctionDesc functionDesc = it.next();
 
-if (cubeFuncs.contains(functionDesc)) {
-it.remove();
-continue;
-}
-
 // let calcite handle count
 if (functionDesc.isCount()) {
 it.remove();

http://git-wip-us.apache.org/repos/asf/kylin/blob/324e4239/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index d73206a..62351d3 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -296,16 +296,20 @@ public class OLAPAggregateRel extends Aggregate 
implements OLAPRel {
 }
 
 private void translateAggregation() {
-// now the realization is known, replace aggregations with what's 
defined on MeasureDesc
-List measures = this.context.realization.getMeasures();
-List newAggrs = 

[20/26] kylin git commit: minor, add more logging

[liyang]

minor, add more logging


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59228300
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59228300
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59228300

Branch: refs/heads/KYLIN-1971
Commit: 59228300fd5f60f415e5430859c111ab39715b55
Parents: d06a425
Author: lidongsjtu 
Authored: Fri Oct 28 14:07:39 2016 +0800
Committer: lidongsjtu 
Committed: Fri Oct 28 14:07:46 2016 +0800

--
 .../org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59228300/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index fa23be1..b96e952 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -103,7 +103,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 dimensionsD.addAll(groupsD);
 dimensionsD.addAll(otherDimsD);
 Cuboid cuboid = Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics);
-logger.info("Cuboid identified: cube={}, cuboidId={}", 
cubeInstance.getName(), cuboid.getId());
+logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, 
otherDimsD={}", cubeInstance.getName(), cuboid.getId(), groupsD, otherDimsD);
 context.setCuboid(cuboid);
 
 // isExactAggregation? meaning: tuples returned from storage requires 
no further aggregation in query engine

[02/26] kylin git commit: KYLIN-2030 fix with KYLIN-1971

[liyang]

KYLIN-2030 fix with KYLIN-1971


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c804dc8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c804dc8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c804dc8d

Branch: refs/heads/KYLIN-1971
Commit: c804dc8d77bb0402d6cd391e582a47f323edbb21
Parents: 324e423
Author: Hongbin Ma 
Authored: Wed Oct 26 14:41:06 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../org/apache/kylin/cube/CubeCapabilityChecker.java   | 13 +++--
 .../org/apache/kylin/metadata/model/ColumnDesc.java|  8 +---
 2 files changed, 4 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c804dc8d/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e509d98..a3c89e5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,22 +25,18 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
-import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.SQLDigest;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,11 +82,8 @@ public class CubeCapabilityChecker {
 } else {
 //for non query-on-facttable 
 if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
-TableDesc tableDesc = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
-Set dimCols = Sets.newHashSet();
-for (ColumnDesc columnDesc : tableDesc.getColumns()) {
-dimCols.add(columnDesc.getRef());
-}
+
+Set dimCols = 
Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
 
 //1. all aggregations on lookup table can be done. For 
distinct count, mark them all DimensionAsMeasures
 // so that the measure has a chance to be upgraded to 
DimCountDistinctMeasureType in 
org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType

http://git-wip-us.apache.org/repos/asf/kylin/blob/c804dc8d/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index a13bd37..e0184b4 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -26,7 +26,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 
 /**
  * Column Metadata from Source. All name should be uppercase.
@@ -203,11 +202,6 @@ public class ColumnDesc implements Serializable {
 
 @Override
 public String toString() {
-return "ColumnDesc{" +
-"id='" + id + '\'' +
-", name='" + name + '\'' +
-", datatype='" + datatype + '\'' +
-", comment='" + comment + '\'' +
-'}';
+return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + 
", datatype='" + datatype + '\'' + ", comment='" + comment + '\'' + '}';
   

[25/26] kylin git commit: minor refactor of CubeSegment

[liyang]

minor refactor of CubeSegment


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/635b3b97
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/635b3b97
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/635b3b97

Branch: refs/heads/KYLIN-1971
Commit: 635b3b97662515b7b9e4c58d6996c28c12fcfb28
Parents: 650f57c
Author: Li Yang 
Authored: Sun Oct 30 15:45:17 2016 +0800
Committer: Li Yang 
Committed: Sun Oct 30 15:45:17 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java | 33 +++-
 1 file changed, 11 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/635b3b97/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index b3d6d6b..1ec01a2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -42,15 +42,13 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
-import javax.annotation.concurrent.GuardedBy;
-
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class CubeSegment implements Comparable, IBuildable, 
ISegment {
 
@@ -117,8 +115,7 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 private Map additionalInfo = new LinkedHashMap();
 
-@GuardedBy("this")
-private Map cuboidBaseShards = Maps.newHashMap(); // cuboid 
id ==> base(starting) shard for this cuboid
+private Map cuboidBaseShards = Maps.newConcurrentMap(); // 
cuboid id ==> base(starting) shard for this cuboid
 
 public CubeDesc getCubeDesc() {
 return getCubeInstance().getDescriptor();
@@ -505,12 +502,9 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public int getTotalShards(long cuboidId) {
 if (totalShards > 0) {
-//shard squashed case
-//logger.info("total shards for {} is {}", cuboidId, totalShards);
 return totalShards;
 } else {
 int ret = getCuboidShardNum(cuboidId);
-//logger.info("total shards for {} is {}", cuboidId, ret);
 return ret;
 }
 }
@@ -519,22 +513,17 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.totalShards = totalShards;
 }
 
-public synchronized short getCuboidBaseShard(Long cuboidId) {
-if (totalShards > 0) {
-//shard squashed case
-
-Short ret = cuboidBaseShards.get(cuboidId);
-if (ret == null) {
-ret = ShardingHash.getShard(cuboidId, totalShards);
-cuboidBaseShards.put(cuboidId, ret);
-}
-
-//logger.info("base for cuboid {} is {}", cuboidId, ret);
-return ret;
-} else {
-//logger.info("base for cuboid {} is {}", cuboidId, 0);
+public short getCuboidBaseShard(Long cuboidId) {
+if (totalShards == 0)
 return 0;
+
+Short ret = cuboidBaseShards.get(cuboidId);
+if (ret == null) {
+ret = ShardingHash.getShard(cuboidId, totalShards);
+cuboidBaseShards.put(cuboidId, ret);
 }
+
+return ret;
 }
 
 public List getBlackoutCuboids() {

[06/26] kylin git commit: KYLIN-2030 enhancement

[liyang]

KYLIN-2030 enhancement


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46284bdc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46284bdc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46284bdc

Branch: refs/heads/KYLIN-1971
Commit: 46284bdca211d59f4a3274f5579dc8fe8a48cc6e
Parents: 1429da5
Author: Hongbin Ma 
Authored: Sat Oct 8 16:14:26 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Oct 27 08:30:13 2016 +0800

--
 .../kylin/cube/CubeCapabilityChecker.java   | 43 +++-
 .../org/apache/kylin/cube/model/CubeDesc.java   |  6 +--
 .../kylin/metadata/model/FunctionDesc.java  |  1 +
 .../test/resources/query/sql_lookup/query06.sql |  1 +
 .../test/resources/query/sql_lookup/query07.sql |  1 +
 5 files changed, 39 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c8460f0..ee21b1c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,18 +25,22 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
-import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.apache.kylin.metadata.realization.SQLDigest;
+import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,20 +70,39 @@ public class CubeCapabilityChecker {
 Collection unmatchedAggregations = 
unmatchedAggregations(aggrFunctions, cube);
 
 // try custom measure types
+// in RAW query, unmatchedDimensions and unmatchedAggregations will 
null, so can't chose RAW cube well!
+//if (!unmatchedDimensions.isEmpty() || 
!unmatchedAggregations.isEmpty()) {
 tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, 
digest, cube, result);
+//}
 
-// try dimension-as-measure
-if (!unmatchedAggregations.isEmpty()) {
-if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+//more tricks
+if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+//for query-on-facttable
+//1. dimension as measure
+
+if (!unmatchedAggregations.isEmpty()) {
 tryDimensionAsMeasures(unmatchedAggregations, digest, cube, 
result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
-} else {
-//deal with query on lookup table, like 
https://issues.apache.org/jira/browse/KYLIN-2030
-if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
-Set dimCols = 
Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
+}
+} else {
+//for non query-on-facttable 
+if 
(cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
+TableDesc tableDesc = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
+Set dimCols = Sets.newHashSet();
+for (ColumnDesc columnDesc : tableDesc.getColumns()) {
+dimCols.add(columnDesc.getRef());
+}
+
+//1. dimension as measure, like max(cal_dt) or count( distinct 
col) from lookup
+if (!unmatchedAggregations.isEmpty()) {
 tryDimensionAsMeasures(unmatchedAggregations, digest, 
cube, result, dimCols);
-} else {
-logger.info("Skip 

[23/26] kylin git commit: KYLIN-2137 add check for 'kylin.job.mr.lib.dir'

[liyang]

KYLIN-2137 add check for 'kylin.job.mr.lib.dir'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5af4435
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5af4435
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5af4435

Branch: refs/heads/KYLIN-1971
Commit: c5af44353d5904ddb8e9107a160872164f0e5f4d
Parents: 8314e74
Author: shaofengshi 
Authored: Fri Oct 28 22:17:04 2016 +0800
Committer: shaofengshi 
Committed: Fri Oct 28 22:17:34 2016 +0800

--
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java| 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c5af4435/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 417d63d..a97a8c7 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -245,7 +245,14 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 
 // for KylinJobMRLibDir
 String mrLibDir = kylinConf.getKylinJobMRLibDir();
-StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
+if (!StringUtils.isBlank(mrLibDir)) {
+File dirFileMRLIB = new File(mrLibDir);
+if (dirFileMRLIB.exists()) {
+StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
+} else {
+logger.info("The directory '" + mrLibDir + "' for 
'kylin.job.mr.lib.dir' does not exist!!!");
+}
+}
 
 setJobTmpJarsAndFiles(job, kylinDependency.toString());
 

[19/26] kylin git commit: KYLIN-1971 fix ITStorageTest

[liyang]

KYLIN-1971 fix ITStorageTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d06a425e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d06a425e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d06a425e

Branch: refs/heads/KYLIN-1971
Commit: d06a425e3b30b52ae895954a9faa034da1ad0e04
Parents: 3b9be19
Author: Li Yang 
Authored: Thu Oct 27 18:03:22 2016 +0800
Committer: Li Yang 
Committed: Thu Oct 27 18:03:22 2016 +0800

--
 .../apache/kylin/metadata/model/TableRef.java   |  8 +++
 .../apache/kylin/storage/StorageMockUtils.java  | 52 +++-
 .../kylin/storage/hbase/ITStorageTest.java  | 18 +++
 3 files changed, 46 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d06a425e/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 254ce18..7089eba 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -70,6 +70,8 @@ public class TableRef {
 return Collections.unmodifiableCollection(columns.values());
 }
 
+// for test only
+@Deprecated
 public TblColRef makeFakeColumn(String name) {
 ColumnDesc colDesc = new ColumnDesc();
 colDesc.setName(name);
@@ -77,6 +79,12 @@ public class TableRef {
 return new TblColRef(this, colDesc);
 }
 
+// for test only
+@Deprecated
+public TblColRef makeFakeColumn(ColumnDesc colDesc) {
+return new TblColRef(this, colDesc);
+}
+
 @Override
 public boolean equals(Object o) {
 if (this == o)

http://git-wip-us.apache.org/repos/asf/kylin/blob/d06a425e/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
--
diff --git 
a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java 
b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
index 456e768..0786f32 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
@@ -28,9 +28,11 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.TupleInfo;
 
@@ -39,7 +41,14 @@ import com.google.common.collect.ImmutableList;
 /**
  */
 public class StorageMockUtils {
-public static TupleInfo newTupleInfo(List groups, 
List aggregations) {
+
+final DataModelDesc model;
+
+public StorageMockUtils(DataModelDesc model) {
+this.model = model;
+}
+
+public TupleInfo newTupleInfo(List groups, List 
aggregations) {
 TupleInfo info = new TupleInfo();
 int idx = 0;
 
@@ -47,34 +56,32 @@ public class StorageMockUtils {
 info.setField(col.getName(), col, idx++);
 }
 
-TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
+TableRef sourceTable = groups.get(0).getTableRef();
 for (FunctionDesc func : aggregations) {
-TblColRef col = func.newFakeRewriteColumn(sourceTable).getRef();
+ColumnDesc colDesc = 
func.newFakeRewriteColumn(sourceTable.getTableDesc());
+TblColRef col = sourceTable.makeFakeColumn(colDesc);
 info.setField(col.getName(), col, idx++);
 }
 
 return info;
 }
 
-public static List buildGroups() {
+public List buildGroups() {
 List groups = new ArrayList();
 
-TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
-TblColRef c1 = TblColRef.mockup(t1, 2, "CAL_DT", "date");
+TblColRef c1 = model.findColumn("DEFAULT.TEST_KYLIN_FACT.CAL_DT");
 groups.add(c1);
 
-TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
-TblColRef c2 = TblColRef.mockup(t2, 14, "META_CATEG_NAME", "string");
+TblColRef c2 

kylin git commit: minor refactor of CubeSegment

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 650f57c88 -> 635b3b976


minor refactor of CubeSegment


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/635b3b97
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/635b3b97
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/635b3b97

Branch: refs/heads/master
Commit: 635b3b97662515b7b9e4c58d6996c28c12fcfb28
Parents: 650f57c
Author: Li Yang 
Authored: Sun Oct 30 15:45:17 2016 +0800
Committer: Li Yang 
Committed: Sun Oct 30 15:45:17 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java | 33 +++-
 1 file changed, 11 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/635b3b97/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index b3d6d6b..1ec01a2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -42,15 +42,13 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
-import javax.annotation.concurrent.GuardedBy;
-
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class CubeSegment implements Comparable, IBuildable, 
ISegment {
 
@@ -117,8 +115,7 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 private Map additionalInfo = new LinkedHashMap();
 
-@GuardedBy("this")
-private Map cuboidBaseShards = Maps.newHashMap(); // cuboid 
id ==> base(starting) shard for this cuboid
+private Map cuboidBaseShards = Maps.newConcurrentMap(); // 
cuboid id ==> base(starting) shard for this cuboid
 
 public CubeDesc getCubeDesc() {
 return getCubeInstance().getDescriptor();
@@ -505,12 +502,9 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public int getTotalShards(long cuboidId) {
 if (totalShards > 0) {
-//shard squashed case
-//logger.info("total shards for {} is {}", cuboidId, totalShards);
 return totalShards;
 } else {
 int ret = getCuboidShardNum(cuboidId);
-//logger.info("total shards for {} is {}", cuboidId, ret);
 return ret;
 }
 }
@@ -519,22 +513,17 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.totalShards = totalShards;
 }
 
-public synchronized short getCuboidBaseShard(Long cuboidId) {
-if (totalShards > 0) {
-//shard squashed case
-
-Short ret = cuboidBaseShards.get(cuboidId);
-if (ret == null) {
-ret = ShardingHash.getShard(cuboidId, totalShards);
-cuboidBaseShards.put(cuboidId, ret);
-}
-
-//logger.info("base for cuboid {} is {}", cuboidId, ret);
-return ret;
-} else {
-//logger.info("base for cuboid {} is {}", cuboidId, 0);
+public short getCuboidBaseShard(Long cuboidId) {
+if (totalShards == 0)
 return 0;
+
+Short ret = cuboidBaseShards.get(cuboidId);
+if (ret == null) {
+ret = ShardingHash.getShard(cuboidId, totalShards);
+cuboidBaseShards.put(cuboidId, ret);
 }
+
+return ret;
 }
 
 public List getBlackoutCuboids() {