kylin git commit: KYLIN-2371 Allow overwrite default spark conf at cube level [Forced Update!]

2017-01-08 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/sparkcubing-rebase 5b7ef354e -> d94434095 (forced update)


KYLIN-2371 Allow overwrite default spark conf at cube level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d9443409
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d9443409
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d9443409

Branch: refs/heads/sparkcubing-rebase
Commit: d9443409542752e9d85dffccd95e54449e26b297
Parents: 1b04c3c
Author: shaofengshi 
Authored: Mon Jan 9 15:23:22 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 15:35:34 2017 +0800

--
 build/conf/kylin-spark-conf.properties  |  5 +--
 build/conf/kylin.properties |  6 ++--
 .../apache/kylin/common/KylinConfigBase.java| 12 ---
 .../kylin/job/constant/ExecutableConstants.java |  1 +
 .../spark/SparkBatchCubingJobBuilder2.java  |  2 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |  7 ++--
 .../kylin/engine/spark/SparkExecutable.java | 38 +---
 .../test_case_data/sandbox/kylin.properties |  3 +-
 8 files changed, 55 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d9443409/build/conf/kylin-spark-conf.properties
--
diff --git a/build/conf/kylin-spark-conf.properties 
b/build/conf/kylin-spark-conf.properties
index 5e6dafe..143e7e4 100644
--- a/build/conf/kylin-spark-conf.properties
+++ b/build/conf/kylin-spark-conf.properties
@@ -1,5 +1,5 @@
 spark.yarn.submit.file.replication=1
-spark.yarn.executor.memoryOverhead=200
+spark.yarn.executor.memoryOverhead=1024
 spark.yarn.driver.memoryOverhead=384
 spark.master=yarn
 spark.submit.deployMode=cluster
@@ -20,7 +20,8 @@ spark.executor.cores=4
 spark.executor.instances=8
 spark.history.kerberos.keytab=none
 spark.history.kerberos.principal=none
-#spark.yarn.jar=hdfs://namenode:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 spark.driver.extraJavaOptions=-Dhdp.version=current
 spark.yarn.am.extraJavaOptions=-Dhdp.version=current
 spark.executor.extraJavaOptions=-Dhdp.version=current
+#spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+#spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec

http://git-wip-us.apache.org/repos/asf/kylin/blob/d9443409/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 54430f0..6efa423 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -131,16 +131,16 @@ kylin.engine.mr.mapper-input-rows=100
 ### Spark Engine ###
 
 # Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run 
spark-submit
-kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
 
 # Spark job submission properties file, default be 
$KYLIN_HOME/conf/kylin-spark-conf.properties
 #kylin.engine.spark.properties-file=
 
 # Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=50
+kylin.engine.spark.rdd-partition-cut-mb=10
 
 # Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=10
+kylin.engine.spark.min-partition=1
 
 # Max partition numbers of rdd
 kylin.engine.spark.max-partition=5000

http://git-wip-us.apache.org/repos/asf/kylin/blob/d9443409/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7d6ac2b..36ddbf4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -695,6 +695,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.engine.mr.config-override.");
 }
 
+public Map getSparkConfigOverride() {
+return getPropertiesByPrefix("kylin.engine.spark.config-override.");
+}
+
 public double getDefaultHadoopJobReducerInputMB() {
 return 
Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
 }
@@ -739,8 +743,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 // ENGINE.SPARK
 // 

 
-public String getSparkHadoopConfDir() {
-return getRequired("kylin.engine.spark.env.hadoop-conf-dir");
+public String getHadoopConfDir() {
+return

kylin git commit: KYLIN-2371 Allow overwrite default spark conf at cube level

2017-01-08 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/sparkcubing-rebase 1b04c3cda -> 5b7ef354e


KYLIN-2371 Allow overwrite default spark conf at cube level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5b7ef354
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5b7ef354
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5b7ef354

Branch: refs/heads/sparkcubing-rebase
Commit: 5b7ef354e96576013a049f9cdbbc863d8afaff32
Parents: 1b04c3c
Author: shaofengshi 
Authored: Mon Jan 9 15:23:22 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 9 15:23:22 2017 +0800

--
 build/conf/kylin-spark-conf.properties  |  5 +--
 build/conf/kylin.properties |  6 ++--
 .../apache/kylin/common/KylinConfigBase.java| 12 ---
 .../kylin/job/constant/ExecutableConstants.java |  1 +
 .../spark/SparkBatchCubingJobBuilder2.java  |  2 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |  7 ++--
 .../kylin/engine/spark/SparkExecutable.java | 37 ++--
 .../test_case_data/sandbox/kylin.properties |  3 +-
 8 files changed, 55 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5b7ef354/build/conf/kylin-spark-conf.properties
--
diff --git a/build/conf/kylin-spark-conf.properties 
b/build/conf/kylin-spark-conf.properties
index 5e6dafe..143e7e4 100644
--- a/build/conf/kylin-spark-conf.properties
+++ b/build/conf/kylin-spark-conf.properties
@@ -1,5 +1,5 @@
 spark.yarn.submit.file.replication=1
-spark.yarn.executor.memoryOverhead=200
+spark.yarn.executor.memoryOverhead=1024
 spark.yarn.driver.memoryOverhead=384
 spark.master=yarn
 spark.submit.deployMode=cluster
@@ -20,7 +20,8 @@ spark.executor.cores=4
 spark.executor.instances=8
 spark.history.kerberos.keytab=none
 spark.history.kerberos.principal=none
-#spark.yarn.jar=hdfs://namenode:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 spark.driver.extraJavaOptions=-Dhdp.version=current
 spark.yarn.am.extraJavaOptions=-Dhdp.version=current
 spark.executor.extraJavaOptions=-Dhdp.version=current
+#spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+#spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec

http://git-wip-us.apache.org/repos/asf/kylin/blob/5b7ef354/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 54430f0..6efa423 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -131,16 +131,16 @@ kylin.engine.mr.mapper-input-rows=100
 ### Spark Engine ###
 
 # Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run 
spark-submit
-kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
 
 # Spark job submission properties file, default be 
$KYLIN_HOME/conf/kylin-spark-conf.properties
 #kylin.engine.spark.properties-file=
 
 # Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=50
+kylin.engine.spark.rdd-partition-cut-mb=10
 
 # Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=10
+kylin.engine.spark.min-partition=1
 
 # Max partition numbers of rdd
 kylin.engine.spark.max-partition=5000

http://git-wip-us.apache.org/repos/asf/kylin/blob/5b7ef354/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7d6ac2b..36ddbf4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -695,6 +695,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.engine.mr.config-override.");
 }
 
+public Map getSparkConfigOverride() {
+return getPropertiesByPrefix("kylin.engine.spark.config-override.");
+}
+
 public double getDefaultHadoopJobReducerInputMB() {
 return 
Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
 }
@@ -739,8 +743,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 // ENGINE.SPARK
 // 

 
-public String getSparkHadoopConfDir() {
-return getRequired("kylin.engine.spark.env.hadoop-conf-dir");
+public String getHadoopConfDir() {
+return

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-08 Thread liyang 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ebe4b5bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ebe4b5bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ebe4b5bb

Branch: refs/heads/yang22-hbase102
Commit: ebe4b5bb0a7dc3bec93268d2cecd2910c33018a0
Parents: 3b70b1a
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Mon Jan 9 06:47:00 2017 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ebe4b5bb/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/5] kylin git commit: KYLIN-2351 fix checkstyle [Forced Update!]

2017-01-08 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 2a875d4c8 -> 04d172e32 (forced update)


KYLIN-2351 fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3396d374
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3396d374
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3396d374

Branch: refs/heads/yang22-hbase102
Commit: 3396d3740281fbcfacfcf22da0657771682e0431
Parents: ee020cc
Author: Billy Liu 
Authored: Fri Jan 6 09:15:38 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 6 09:16:52 2017 +0800

--
 .../src/test/java/org/apache/kylin/query/ITMassInQueryTest.java | 1 -
 .../src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java | 1 -
 2 files changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3396d374/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 18d79ae..fd3abfb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -26,7 +26,6 @@ import java.util.Random;
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/kylin/blob/3396d374/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
index 786d7d1..605b79e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
@@ -20,7 +20,6 @@ package org.apache.kylin.storage.hbase;
 
 import java.io.IOException;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-08 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/ebe4b5bb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ebe4b5bb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[5/5] kylin git commit: KYLIN-2233 Support HBase 1.0.2

2017-01-08 Thread liyang 
KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/04d172e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/04d172e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/04d172e3

Branch: refs/heads/yang22-hbase102
Commit: 04d172e3288ea1e3495a228981d74f59af486e38
Parents: ebe4b5b
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Li Yang 
Committed: Mon Jan 9 06:56:27 2017 +

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/04d172e3/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/04d172e3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/04d172e3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of

kylin git commit: KYLIN-2368 Enable Findbugs plugin

2017-01-08 Thread billyliu 
Repository: kylin
Updated Branches:
  refs/heads/master 6f8cdeb55 -> 75625ae51


KYLIN-2368 Enable Findbugs plugin


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75625ae5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75625ae5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75625ae5

Branch: refs/heads/master
Commit: 75625ae512b3d6cc080053e4e8871f3fff816ef0
Parents: 6f8cdeb
Author: Billy Liu 
Authored: Mon Jan 9 15:02:06 2017 +0800
Committer: Billy Liu 
Committed: Mon Jan 9 15:02:13 2017 +0800

--
 atopcalcite/pom.xml |  4 +--
 core-common/pom.xml |  5 +--
 core-cube/pom.xml   |  2 +-
 core-dictionary/pom.xml |  5 +--
 core-job/pom.xml|  5 +--
 core-metadata/pom.xml   |  5 +--
 core-storage/pom.xml|  5 +--
 engine-mr/pom.xml   |  6 +---
 engine-spark/pom.xml| 16 --
 jdbc/pom.xml|  3 +-
 kylin-it/pom.xml| 11 ---
 pom.xml | 73 +++-
 query/pom.xml   |  1 +
 server-base/pom.xml |  5 +--
 server/pom.xml  |  2 +-
 source-hive/pom.xml |  5 +--
 source-kafka/pom.xml|  2 +-
 storage-hbase/pom.xml   |  5 +--
 tomcat-ext/pom.xml  |  8 +++--
 tool/pom.xml| 11 ---
 20 files changed, 92 insertions(+), 87 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b1bc171..b916df2 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -24,6 +24,7 @@
 atopcalcite
 jar
 Apache Kylin - Calcite Overrides
+Apache Kylin - Calcite Overrides
 
 
 org.apache.kylin
@@ -31,9 +32,6 @@
 2.0.0-SNAPSHOT
 
 
-
-
-
 
 
 org.apache.calcite

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 2265651..25b10a7 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-common
 jar
 Apache Kylin - Core Common
+Apache Kylin - Core Common
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
-
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index b7db363..0a30432 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -24,12 +24,12 @@
 kylin-core-cube
 jar
 Apache Kylin - Core Cube
+Apache Kylin - Core Cube
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index a74eccb..5d839dd 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-dictionary
 jar
 Apache Kylin - Core Dictionary
+Apache Kylin - Core Dictionary
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
-
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index dad0428..36d34c8 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-job
 jar
 Apache Kylin - Core Job
+Apache Kylin - Core Job
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
-
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index f5c8f8a..b3fe885 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -24,17 +24,14 @@
 kylin-core-metadata
 jar
 Apache Kylin - Core Metadata
+Apache Kylin - Core Metadata
 
 
 org.apache.kylin
 kylin
 2.0.0-SNAPSHOT
-
 
 
-
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 8987608..9bd3f04 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@

[1/5] kylin git commit: KYLIN-2351 fix checkstyle [Forced Update!]

2017-01-08 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 c7edb8ff2 -> 07a70eceb (forced update)


KYLIN-2351 fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3396d374
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3396d374
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3396d374

Branch: refs/heads/yang22-cdh5.7
Commit: 3396d3740281fbcfacfcf22da0657771682e0431
Parents: ee020cc
Author: Billy Liu 
Authored: Fri Jan 6 09:15:38 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 6 09:16:52 2017 +0800

--
 .../src/test/java/org/apache/kylin/query/ITMassInQueryTest.java | 1 -
 .../src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java | 1 -
 2 files changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3396d374/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 18d79ae..fd3abfb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -26,7 +26,6 @@ import java.util.Random;
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/kylin/blob/3396d374/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
index 786d7d1..605b79e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
@@ -20,7 +20,6 @@ package org.apache.kylin.storage.hbase;
 
 import java.io.IOException;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

[1/4] kylin git commit: KYLIN-2351 fix checkstyle [Forced Update!]

2017-01-08 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 7558802d3 -> ebe4b5bb0 (forced update)


KYLIN-2351 fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3396d374
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3396d374
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3396d374

Branch: refs/heads/yang22-hbase1.x
Commit: 3396d3740281fbcfacfcf22da0657771682e0431
Parents: ee020cc
Author: Billy Liu 
Authored: Fri Jan 6 09:15:38 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 6 09:16:52 2017 +0800

--
 .../src/test/java/org/apache/kylin/query/ITMassInQueryTest.java | 1 -
 .../src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java | 1 -
 2 files changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3396d374/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 18d79ae..fd3abfb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -26,7 +26,6 @@ import java.util.Random;
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/kylin/blob/3396d374/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
index 786d7d1..605b79e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
@@ -20,7 +20,6 @@ package org.apache.kylin.storage.hbase;
 
 import java.io.IOException;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-08 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/ebe4b5bb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ebe4b5bb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[2/4] kylin git commit: KYLIN-2367 fix select * where ...

2017-01-08 Thread liyang 
KYLIN-2367 fix select * where ...


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3b70b1a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3b70b1a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3b70b1a8

Branch: refs/heads/yang22-hbase1.x
Commit: 3b70b1a8b3efda3c6f85f841786e3dece9ece848
Parents: 3396d37
Author: Li Yang 
Authored: Mon Jan 9 14:41:05 2017 +0800
Committer: Li Yang 
Committed: Mon Jan 9 14:43:40 2017 +0800

--
 .../org/apache/calcite/sql2rel/SqlToRelConverter.java | 14 +-
 1 file changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3b70b1a8/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index cf36f61..ae8194c 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -567,7 +567,8 @@ public class SqlToRelConverter {
  * 
  *   LogicalSort (optional)
  *|- LogicalProject
- *|- OLAPTableScan
+ *|- LogicalFilter (optional)
+ *|- OLAPTableScan
  */
 LogicalProject rootPrj = null;
 LogicalSort rootSort = null;
@@ -580,11 +581,14 @@ public class SqlToRelConverter {
 return root;
 }
 
-if 
(!rootPrj.getInput().getClass().getSimpleName().equals("OLAPTableScan"))
+RelNode input = rootPrj.getInput();
+if (!(//
+input.getClass().getSimpleName().equals("OLAPTableScan")//
+|| (input.getClass().getSimpleName().equals("LogicalFilter") 
&& input.getInput(0).getClass().getSimpleName().equals("OLAPTableScan"))//
+ ))
 return root;
 
-RelNode scan = rootPrj.getInput();
-if (rootPrj.getRowType().getFieldCount() < 
scan.getRowType().getFieldCount())
+if (rootPrj.getRowType().getFieldCount() < 
input.getRowType().getFieldCount())
 return root;
 
 RelDataType inType = rootPrj.getRowType();
@@ -603,7 +607,7 @@ public class SqlToRelConverter {
 }
 
 RelDataType projRowType = 
getCluster().getTypeFactory().createStructType(projTypeBuilder);
-rootPrj = LogicalProject.create(scan, projExp, projRowType);
+rootPrj = LogicalProject.create(input, projExp, projRowType);
 if (rootSort != null) {
 rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), 
rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch);
 }

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-08 Thread liyang 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ebe4b5bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ebe4b5bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ebe4b5bb

Branch: refs/heads/yang22-hbase1.x
Commit: ebe4b5bb0a7dc3bec93268d2cecd2910c33018a0
Parents: 3b70b1a
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Mon Jan 9 06:47:00 2017 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ebe4b5bb/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

kylin git commit: KYLIN-2367 fix select * where ...

2017-01-08 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang22 3396d3740 -> 3b70b1a8b


KYLIN-2367 fix select * where ...


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3b70b1a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3b70b1a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3b70b1a8

Branch: refs/heads/yang22
Commit: 3b70b1a8b3efda3c6f85f841786e3dece9ece848
Parents: 3396d37
Author: Li Yang 
Authored: Mon Jan 9 14:41:05 2017 +0800
Committer: Li Yang 
Committed: Mon Jan 9 14:43:40 2017 +0800

--
 .../org/apache/calcite/sql2rel/SqlToRelConverter.java | 14 +-
 1 file changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3b70b1a8/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index cf36f61..ae8194c 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -567,7 +567,8 @@ public class SqlToRelConverter {
  * 
  *   LogicalSort (optional)
  *|- LogicalProject
- *|- OLAPTableScan
+ *|- LogicalFilter (optional)
+ *|- OLAPTableScan
  */
 LogicalProject rootPrj = null;
 LogicalSort rootSort = null;
@@ -580,11 +581,14 @@ public class SqlToRelConverter {
 return root;
 }
 
-if 
(!rootPrj.getInput().getClass().getSimpleName().equals("OLAPTableScan"))
+RelNode input = rootPrj.getInput();
+if (!(//
+input.getClass().getSimpleName().equals("OLAPTableScan")//
+|| (input.getClass().getSimpleName().equals("LogicalFilter") 
&& input.getInput(0).getClass().getSimpleName().equals("OLAPTableScan"))//
+ ))
 return root;
 
-RelNode scan = rootPrj.getInput();
-if (rootPrj.getRowType().getFieldCount() < 
scan.getRowType().getFieldCount())
+if (rootPrj.getRowType().getFieldCount() < 
input.getRowType().getFieldCount())
 return root;
 
 RelDataType inType = rootPrj.getRowType();
@@ -603,7 +607,7 @@ public class SqlToRelConverter {
 }
 
 RelDataType projRowType = 
getCluster().getTypeFactory().createStructType(projTypeBuilder);
-rootPrj = LogicalProject.create(scan, projExp, projRowType);
+rootPrj = LogicalProject.create(input, projExp, projRowType);
 if (rootSort != null) {
 rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), 
rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch);
 }

[3/3] kylin git commit: Upgrade Calcite to 1.11 and Avatica to 1.9

2017-01-08 Thread billyliu 
Upgrade Calcite to 1.11 and Avatica to 1.9


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a2117d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a2117d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a2117d4

Branch: refs/heads/calcite-11
Commit: 3a2117d4cd79c7d4aa0de60bd2c847d30ad5a5c9
Parents: 68ff75d
Author: Billy Liu 
Authored: Mon Jan 9 13:33:09 2017 +0800
Committer: Billy Liu 
Committed: Mon Jan 9 13:33:09 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 3941 +++---
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 pom.xml |4 +-
 3 files changed, 2402 insertions(+), 1545 deletions(-)
--

[2/3] kylin git commit: Upgrade Calcite to 1.11 and Avatica to 1.9

2017-01-08 Thread billyliu 
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a2117d4/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 7da3040..31bc4cd 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -16,24 +16,6 @@
  */
 package org.apache.calcite.sql2rel;
 
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
 import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.plan.Convention;
@@ -85,7 +67,6 @@ import org.apache.calcite.rel.stream.Delta;
 import org.apache.calcite.rel.stream.LogicalDelta;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -116,6 +97,7 @@ import org.apache.calcite.sql.SqlCallBinding;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlDelete;
 import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainFormat;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql.SqlFunction;
 import org.apache.calcite.sql.SqlIdentifier;
@@ -177,17 +159,37 @@ import org.apache.calcite.util.NumberUtil;
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 import org.apache.calcite.util.trace.CalciteTrace;
-import org.slf4j.Logger;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
 
 /*
  * The code has synced with calcite. Hope one day, we could remove the 
hardcode override point.
@@ -209,42 +211,56 @@ import com.google.common.collect.Sets;
 public class SqlToRelConverter {
 //~ Static fields/initializers 
-
 
+protected static final Logger SQL2REL_LOGGER =
+CalciteTrace.getSqlToRelTracer();
+
+private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
 /** Size of the smallest IN list that will be converted to a semijoin to a
  * static table. */
-public static final int DEFAULT_IN_SUBQUERY_THRESHOLD = 20;
-protected static final Logger SQL2REL_LOGGER = 
CalciteTrace.getSqlToRelTracer();
-private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+
+@Deprecated // to be removed before 2.0
+public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
+DEFAULT_IN_SUB_QUERY_THRESHOLD;
 
 //~ Instance fields 

-public final SqlToRelConverter.Config config;
-public final RelOptTable.ViewExpander viewExpander;
+
 protected final SqlValidator validator;
 protected final RexBuilder rexBuilder;
 protected final Prepare.CatalogReader catalogReader;
 protected final RelOptCluster cluster;
+private DefaultValueFactory defaultValueFactory;
+private SubQueryConverter subQueryConverter;
 protected final List leaves = new ArrayList<>();
-protected final RelDataTypeFactory typeFactory;
 private final List dynamicParamSqlNodes = new 
ArrayList<>();

kylin git commit: minor, enable jacoco plugin

2017-01-08 Thread billyliu 
Repository: kylin
Updated Branches:
  refs/heads/master e46d699e0 -> 68ff75de2


minor, enable jacoco plugin


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68ff75de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68ff75de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68ff75de

Branch: refs/heads/master
Commit: 68ff75de260e6e93197103c0fd06009cf73c8e21
Parents: e46d699
Author: Billy Liu 
Authored: Mon Jan 9 00:29:57 2017 +0800
Committer: Billy Liu 
Committed: Mon Jan 9 00:29:57 2017 +0800

--
 pom.xml | 47 +++
 1 file changed, 31 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68ff75de/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 22681e8..d883eca 100644
--- a/pom.xml
+++ b/pom.xml
@@ -922,19 +922,7 @@
 
 org.jacoco
 jacoco-maven-plugin
-0.7.7.201606060606
-
-${sonar.jacoco.reportPath}
-true
-
-
-
-agent
-
-prepare-agent
-
-
-
+0.7.8
 
 
 org.apache.maven.plugins
@@ -1079,6 +1067,35 @@
 
 
 
+org.jacoco
+jacoco-maven-plugin
+
+true
+
+${sonar.jacoco.reportPath}
+
+
+
+
+pre-test
+
+prepare-agent
+
+
+
surefireArgLine
+
+
+
+post-test
+test
+
+report
+
+
+
+
+
+
 org.apache.maven.plugins
 maven-surefire-plugin
 2.19.1
@@ -1101,11 +1118,9 @@
 
file:${project.basedir}/../build/conf/kylin-tools-log4j.properties
 
 
-
-javaagent:${project.build.testOutputDirectory}/jamm.jar 
${argLine}
+
-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} 
${surefireArgLine}
 
 
-
-