[1/2] kylin git commit: minor,update model designer

[zhongjian]

Repository: kylin
Updated Branches:
  refs/heads/master 94acbc713 -> 76e321b2d


minor,update model designer

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc1a49ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc1a49ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc1a49ec

Branch: refs/heads/master
Commit: bc1a49ec57987cf52d2596ac4d7a439b7596eb0d
Parents: 94acbc7
Author: chenzhx <346839...@qq.com>
Authored: Wed Dec 28 11:11:08 2016 +0800
Committer: zhongjian 
Committed: Wed Dec 28 15:15:36 2016 +0800

--
 webapp/app/js/controllers/modelDataModel.js |  5 
 webapp/app/js/controllers/modelSchema.js| 29 ++--
 webapp/app/js/services/tree.js  | 20 --
 webapp/app/partials/cubes/cube_clone.html   | 23 
 .../partials/modelDesigner/model_measures.html  |  1 +
 webapp/app/partials/models/model_clone.html | 10 ---
 6 files changed, 20 insertions(+), 68 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/controllers/modelDataModel.js
--
diff --git a/webapp/app/js/controllers/modelDataModel.js 
b/webapp/app/js/controllers/modelDataModel.js
index 95c42c4..07009a2 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -203,12 +203,17 @@ KylinApp.controller('ModelDataModelCtrl', function 
($location,$scope, $modal,cub
 modelsManager.selectedModel.dimensions.splice(i, 
1);
 }
 }
+delete $scope.aliasTableMap[lookup.alias];
 lookupList.splice(lookupList.indexOf(lookup), 1);
+
$scope.aliasName.splice($scope.aliasName.indexOf(lookup.alias),1);
 }
 });
 }else{
+delete $scope.aliasTableMap[lookup.alias];
 lookupList.splice(lookupList.indexOf(lookup), 1);
+$scope.aliasName.splice($scope.aliasName.indexOf(lookup.alias),1);
 }
+console.log($scope.aliasName);
 };
 
 $scope.changeKey = function(index){

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/controllers/modelSchema.js
--
diff --git a/webapp/app/js/controllers/modelSchema.js 
b/webapp/app/js/controllers/modelSchema.js
index e309873..f917f1e 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -249,35 +249,10 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, 
QueryService, UserServi
* 1.metric can't be null
*/
   $scope.check_model_measure = function () {
-
-var errors = [];
-if (!modelsManager.selectedModel.metrics || 
!modelsManager.selectedModel.metrics.length) {
-  errors.push("Please define your metrics.");
-}
-var errorInfo = "";
-angular.forEach(errors, function (item) {
-  errorInfo += "\n" + item;
-});
-if (errors.length) {
-  SweetAlert.swal('', errorInfo, 'warning');
-  return false;
-} else {
-  return true;
-}
-
+return true;
   };
   $scope.check_model_setting = function () {
-var errors = [];
-var errorInfo = "";
-angular.forEach(errors, function (item) {
-  errorInfo += "\n" + item;
-});
-if (errors.length) {
-  SweetAlert.swal('', errorInfo, 'warning');
-  return false;
-} else {
-  return true;
-}
+return true;
   }
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc1a49ec/webapp/app/js/services/tree.js
--
diff --git a/webapp/app/js/services/tree.js b/webapp/app/js/services/tree.js
index 20de19b..921aefe 100755
--- a/webapp/app/js/services/tree.js
+++ b/webapp/app/js/services/tree.js
@@ -72,27 +72,7 @@ KylinApp.service('ModelGraphService', function (VdmUtil) {
   }
 });
 
-/*  angular.forEach(model.dimensions, function (dimension, index) {
-// for dimension on lookup table
-if(model.fact_table!==dimension.table){
-var lookup = _.find(graphData.children,function(item){
-  return item.name === dimension.table;
-});
 
-  angular.forEach(lookup.join.primary_key, function(pk, index){
-  for (var i = 0; i < lookup._children.length; i++) {
-  if(lookup._children[i].name == pk)
-  break;
-  }
-  if(i == lookup._children.length) {
-lookup._children.push({
-  

kylin git commit: minor,refine rootfact select show width database

[zhongjian]

Repository: kylin
Updated Branches:
  refs/heads/master 8747e24da -> 94acbc713


minor,refine rootfact select show width database

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94acbc71
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94acbc71
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94acbc71

Branch: refs/heads/master
Commit: 94acbc7130bb9084af9b5ac6def2527c5d238b15
Parents: 8747e24
Author: luguosheng <550175...@qq.com>
Authored: Wed Dec 28 11:39:05 2016 +0800
Committer: zhongjian 
Committed: Wed Dec 28 15:09:17 2016 +0800

--
 webapp/app/partials/modelDesigner/data_model.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/94acbc71/webapp/app/partials/modelDesigner/data_model.html
--
diff --git a/webapp/app/partials/modelDesigner/data_model.html 
b/webapp/app/partials/modelDesigner/data_model.html
index 6e66e91..4d72266 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -27,7 +27,7 @@
 
 
   

kylin git commit: minor, invoke toString of translated filter may be expensive

[gaodayue]

Repository: kylin
Updated Branches:
  refs/heads/master 9898a07b1 -> 7aabe4cb1


minor, invoke toString of translated filter may be expensive


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7aabe4cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7aabe4cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7aabe4cb

Branch: refs/heads/master
Commit: 7aabe4cb1d1480f99b4724b20598ba216f44586a
Parents: 9898a07
Author: gaodayue 
Authored: Wed Dec 28 12:20:51 2016 +0800
Committer: gaodayue 
Committed: Wed Dec 28 12:20:51 2016 +0800

--
 .../java/org/apache/kylin/dict/BuiltInFunctionTransformer.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7aabe4cb/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index f451c2c..6ef2f96 100755
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -57,13 +57,13 @@ public class BuiltInFunctionTransformer implements 
ITupleFilterTransformer {
 //normal case
 translated = translateCompareTupleFilter((CompareTupleFilter) 
tupleFilter);
 if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {" 
+ translated + "}");
+logger.debug("Translated {{}} to IN clause: {{}}", 
tupleFilter, translated);
 }
 } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
 //like case
 translated = 
translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
 if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {" 
+ translated + "}");
+logger.debug("Translated {{}} to IN clause: {{}}", 
tupleFilter, translated);
 }
 } else if (tupleFilter instanceof LogicalTupleFilter) {
 @SuppressWarnings("unchecked")

[1/4] kylin git commit: KYLIN-2138 bug fix again [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 7b84e7e7c -> 58180883a (forced update)


KYLIN-2138 bug fix again


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e0e9a64
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e0e9a64
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e0e9a64

Branch: refs/heads/yang22-hbase102
Commit: 1e0e9a64f07eb0d53ef5f58850b6e9baf1e36d6c
Parents: 37dffd7
Author: Hongbin Ma 
Authored: Wed Dec 28 10:22:33 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:22:33 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0e9a64/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index de28b17..bc644cc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,7 +327,8 @@ public class QueryService extends BasicService {
 }
 
 final String queryId = UUID.randomUUID().toString();
-BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
+if (sqlRequest.getBackdoorToggles() != null)
+BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
 QueryContext.setQueryId(queryId);
 
 try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f3f0b9d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f3f0b9d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f3f0b9d

Branch: refs/heads/yang22-hbase102
Commit: 3f3f0b9d66513f880c4f572f654a23aa6f85341a
Parents: 1e0e9a6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:24:55 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
- 

[4/4] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[liyang]

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58180883
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58180883
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58180883

Branch: refs/heads/yang22-hbase102
Commit: 58180883a92a4ba100ad3f904c40cb3e2e476625
Parents: 3f3f0b9
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:28:14 2016 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58180883/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/58180883/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/58180883/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of 

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb6a5545
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb6a5545
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb6a5545

Branch: refs/heads/yang22-cdh5.7
Commit: eb6a55457e74b500d35b4e48dc06219a899da27e
Parents: 3f3f0b9
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:26:30 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6a5545/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6a5545/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6a5545/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f3f0b9d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f3f0b9d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f3f0b9d

Branch: refs/heads/yang22-cdh5.7
Commit: 3f3f0b9d66513f880c4f572f654a23aa6f85341a
Parents: 1e0e9a6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:24:55 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-   

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[1/4] kylin git commit: KYLIN-2138 bug fix again [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 9c5554db1 -> eb6a55457 (forced update)


KYLIN-2138 bug fix again


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e0e9a64
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e0e9a64
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e0e9a64

Branch: refs/heads/yang22-cdh5.7
Commit: 1e0e9a64f07eb0d53ef5f58850b6e9baf1e36d6c
Parents: 37dffd7
Author: Hongbin Ma 
Authored: Wed Dec 28 10:22:33 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:22:33 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0e9a64/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index de28b17..bc644cc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,7 +327,8 @@ public class QueryService extends BasicService {
 }
 
 final String queryId = UUID.randomUUID().toString();
-BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
+if (sqlRequest.getBackdoorToggles() != null)
+BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
 QueryContext.setQueryId(queryId);
 
 try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[1/3] kylin git commit: KYLIN-2138 bug fix again [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x be8e310ae -> 3f3f0b9d6 (forced update)


KYLIN-2138 bug fix again


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e0e9a64
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e0e9a64
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e0e9a64

Branch: refs/heads/yang22-hbase1.x
Commit: 1e0e9a64f07eb0d53ef5f58850b6e9baf1e36d6c
Parents: 37dffd7
Author: Hongbin Ma 
Authored: Wed Dec 28 10:22:33 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:22:33 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0e9a64/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index de28b17..bc644cc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,7 +327,8 @@ public class QueryService extends BasicService {
 }
 
 final String queryId = UUID.randomUUID().toString();
-BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
+if (sqlRequest.getBackdoorToggles() != null)
+BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
 QueryContext.setQueryId(queryId);
 
 try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f3f0b9d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f3f0b9d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f3f0b9d

Branch: refs/heads/yang22-hbase1.x
Commit: 3f3f0b9d66513f880c4f572f654a23aa6f85341a
Parents: 1e0e9a6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:24:55 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f3f0b9d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
- 

kylin git commit: KYLIN-2138 bug fix again

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22 37dffd708 -> 1e0e9a64f


KYLIN-2138 bug fix again


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e0e9a64
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e0e9a64
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e0e9a64

Branch: refs/heads/yang22
Commit: 1e0e9a64f07eb0d53ef5f58850b6e9baf1e36d6c
Parents: 37dffd7
Author: Hongbin Ma 
Authored: Wed Dec 28 10:22:33 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:22:33 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0e9a64/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index de28b17..bc644cc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,7 +327,8 @@ public class QueryService extends BasicService {
 }
 
 final String queryId = UUID.randomUUID().toString();
-BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
+if (sqlRequest.getBackdoorToggles() != null)
+BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
 QueryContext.setQueryId(queryId);
 
 try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {

kylin git commit: KYLIN-2318 bug fix again...

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master cccf3a9c1 -> 9898a07b1


KYLIN-2318 bug fix again...


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9898a07b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9898a07b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9898a07b

Branch: refs/heads/master
Commit: 9898a07b1451c29b25fb295b22ed62015073df93
Parents: cccf3a9
Author: Hongbin Ma 
Authored: Wed Dec 28 10:20:31 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Dec 28 10:21:28 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9898a07b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index de28b17..bc644cc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,7 +327,8 @@ public class QueryService extends BasicService {
 }
 
 final String queryId = UUID.randomUUID().toString();
-BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
+if (sqlRequest.getBackdoorToggles() != null)
+BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
 QueryContext.setQueryId(queryId);
 
 try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {

kylin git commit: fix NPE in OLAPEnumerator.setConnectionProperties()

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 84ec908b5 -> cccf3a9c1


fix NPE in OLAPEnumerator.setConnectionProperties()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cccf3a9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cccf3a9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cccf3a9c

Branch: refs/heads/master
Commit: cccf3a9c1a8dd0c7ac326c958aafab050a3886cb
Parents: 84ec908
Author: Li Yang 
Authored: Wed Dec 28 09:45:18 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 28 09:45:18 2016 +0800

--
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cccf3a9c/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index e4fc937..f012638 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -25,6 +25,7 @@ import java.util.Properties;
 import org.apache.calcite.DataContext;
 import org.apache.calcite.jdbc.CalciteConnection;
 import org.apache.calcite.linq4j.Enumerator;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
@@ -161,8 +162,10 @@ public class OLAPEnumerator implements 
Enumerator {
 Properties connProps = conn.getProperties();
 
 String propThreshold = 
connProps.getProperty(OLAPQuery.PROP_SCAN_THRESHOLD);
-int threshold = Integer.valueOf(propThreshold);
-olapContext.storageContext.setThreshold(threshold);
+if (!StringUtils.isBlank(propThreshold)) {
+int threshold = Integer.valueOf(propThreshold);
+olapContext.storageContext.setThreshold(threshold);
+}
 }
 
 }

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 4c4559f7f -> 7b84e7e7c (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/be8e310a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/be8e310a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be8e310a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be8e310a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be8e310a

Branch: refs/heads/yang22-hbase102
Commit: be8e310ae5b79ae64eca86b39303869554afec5f
Parents: 37dffd7
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 28 08:47:25 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/be8e310a/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-   

[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[billyliu]

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b84e7e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b84e7e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b84e7e7

Branch: refs/heads/yang22-hbase102
Commit: 7b84e7e7c5a6b8078938e42e5912a613eb8a0fbb
Parents: be8e310
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 28 08:52:42 2016 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b84e7e7/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b84e7e7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b84e7e7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of 

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 cb23ab8b8 -> 9c5554db1 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/be8e310a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/be8e310a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 

[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[billyliu]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c5554db
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c5554db
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c5554db

Branch: refs/heads/yang22-cdh5.7
Commit: 9c5554db12d76babd9281b736d1d2648f4715f8b
Parents: be8e310
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Wed Dec 28 08:49:08 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c5554db/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/9c5554db/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9c5554db/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x e9465c2ed -> be8e310ae (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/be8e310a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/be8e310a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be8e310a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be8e310a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be8e310a

Branch: refs/heads/yang22-hbase1.x
Commit: be8e310ae5b79ae64eca86b39303869554afec5f
Parents: 37dffd7
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 28 08:47:25 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/be8e310a/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-   

kylin git commit: KYLIN-2327 fix UT

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 3e0edc842 -> 84ec908b5


KYLIN-2327 fix UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/84ec908b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/84ec908b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/84ec908b

Branch: refs/heads/master
Commit: 84ec908b57cff0d3167cc35a9e4ea6d4b7bf94ce
Parents: 3e0edc8
Author: Billy Liu 
Authored: Wed Dec 28 08:42:33 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 28 08:42:33 2016 +0800

--
 .../src/test/java/org/apache/kylin/common/util/BasicTest.java   | 1 -
 kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java| 1 -
 2 files changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/84ec908b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index bbaab09..5512147b 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,7 +34,6 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/kylin/blob/84ec908b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java 
b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 3ae7521..4f1f591 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -48,7 +48,6 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.query.enumerator.OLAPQuery;
 import org.apache.kylin.query.relnode.OLAPContext;
 import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.query.schema.OLAPSchemaFactory;

[1/2] kylin git commit: KYLIN-2327 enable check-style for test code

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 1563eb945 -> 3e0edc842


http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
deleted file mode 100644
index 2edf11e..000
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements.  See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership.  The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License.  You may obtain a copy of the License at
-// * 
-// * http://www.apache.org/licenses/LICENSE-2.0
-// * 
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-//*/
-//
-//package org.apache.kylin.storage.minicluster;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.sql.Connection;
-//import java.sql.DriverManager;
-//import java.sql.SQLException;
-//
-//import org.apache.commons.io.FileUtils;
-//import org.apache.hadoop.hdfs.MiniDFSCluster;
-//import org.apache.hadoop.hive.conf.HiveConf;
-//import org.apache.hadoop.hive.service.HiveInterface;
-//import org.apache.hadoop.mapred.JobConf;
-//import org.apache.hadoop.mapred.MiniMRCluster;
-//
-///**
-// * This is a test case to verify whether the query can be executed on Hive 
minicluster;
-// * You need set $HADOOP_HOME environment parameter before run it;
-// * @author shaoshi
-// *
-// */
-//public class HiveMiniClusterTest extends HiveJDBCClientTest {
-//public static final File HIVE_BASE_DIR = new File("target/hive");
-//public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR + 
"/scratchdir");
-//public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR 
+ "/localscratchdir");
-//public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR + 
"/metastoredb");
-//public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR + 
"/logs");
-//public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp");
-//public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + 
"/warehouse");
-//public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + 
"/testdata");
-//public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + 
"/hadooptmp");
-//protected HiveInterface client;
-//
-//protected MiniDFSCluster miniDFS;
-//protected MiniMRCluster miniMR;
-//
-////@Before
-//public void setup() {
-//super.setup();
-//startHiveMiniCluster();
-//}
-//
-//protected void startHiveMiniCluster() {
-////Create and configure location for hive to dump junk in target 
folder
-//try {
-//FileUtils.forceMkdir(HIVE_BASE_DIR);
-//FileUtils.forceMkdir(HIVE_SCRATCH_DIR);
-//FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR);
-//FileUtils.forceMkdir(HIVE_LOGS_DIR);
-//FileUtils.forceMkdir(HIVE_TMP_DIR);
-//FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR);
-//FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR);
-//FileUtils.forceMkdir(HIVE_TESTDATA_DIR);
-//} catch (IOException e1) {
-//e1.printStackTrace();
-//System.exit(1);
-//}
-//
-//System.setProperty("javax.jdo.option.ConnectionURL", 
"jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() + 
";create=true");
-//System.setProperty("hive.metastore.warehouse.dir", 
HIVE_WAREHOUSE_DIR.getAbsolutePath());
-//System.setProperty("hive.exec.scratchdir", 
HIVE_SCRATCH_DIR.getAbsolutePath());
-//System.setProperty("hive.exec.local.scratchdir", 
HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath());
-//System.setProperty("hive.metastore.metadb.dir", 
HIVE_METADB_DIR.getAbsolutePath());
-//System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath());
-//System.setProperty("hive.querylog.location", 
HIVE_TMP_DIR.getAbsolutePath());
-//System.setProperty("hadoop.tmp.dir", 
HIVE_HADOOP_TMP_DIR.getAbsolutePath());
-//System.setProperty("derby.stream.error.file", 
HIVE_BASE_DIR.getAbsolutePath() + "/derby.log");
-//
-// 

[2/2] kylin git commit: KYLIN-2327 enable check-style for test code

[billyliu]

KYLIN-2327 enable check-style for test code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3e0edc84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3e0edc84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3e0edc84

Branch: refs/heads/master
Commit: 3e0edc842ec4f6f7d9ef04dfa26afce101d2fd6e
Parents: 1563eb9
Author: Billy Liu 
Authored: Wed Dec 28 01:16:35 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 28 01:16:58 2016 +0800

--
 .../apache/kylin/common/KylinConfigTest.java|   6 +-
 .../org/apache/kylin/common/util/BasicTest.java |   2 +-
 .../apache/kylin/common/util/JacksonBean.java   |   7 +-
 .../apache/kylin/common/util/JacksonTest.java   |   5 +-
 .../org/apache/kylin/cube/CubeDescTest.java |   6 +-
 .../org/apache/kylin/cube/CubeManagerTest.java  |  16 +-
 .../inmemcubing/ConcurrentDiskStoreTest.java|  27 +-
 .../cube/inmemcubing/MemDiskStoreTest.java  |  27 +-
 .../inmemcubing/MemoryBudgetControllerTest.java |  27 +-
 .../validation/rule/DictionaryRuleTest.java |   1 -
 .../gridtable/AggregationCacheMemSizeTest.java  |  42 +-
 .../kylin/gridtable/SimpleGridTableTest.java|  27 +-
 .../gridtable/SimpleInvertedIndexTest.java  |  27 +-
 .../metadata/measure/MeasureCodecTest.java  |   6 +-
 .../kylin/dict/AppendTrieDictionaryTest.java|   3 +-
 .../apache/kylin/dict/CachedTreeMapTest.java|  21 +-
 .../kylin/dict/DictionaryManagerTest.java   |   3 +-
 .../kylin/dict/TrieDictionaryForestTest.java|   2 +-
 .../kylin/dict/lookup/LookupTableTest.java  |   1 -
 .../impl/threadpool/DefaultSchedulerTest.java   |  32 +-
 .../measure/AggregatorMemEstimateTest.java  |  17 +-
 .../kylin/measure/hllc/HLLCounterOldTest.java   |   3 +-
 .../kylin/measure/hllc/HLLCounterTest.java  |  13 +-
 .../measure/topn/TopNCounterBasicTest.java  |  16 +-
 .../topn/TopNCounterCombinationTest.java|  16 +-
 .../measure/topn/TopNCounterSerializerTest.java |   2 -
 .../kylin/measure/topn/TopNCounterTest.java |  16 +-
 .../kylin/metadata/MetadataManagerTest.java |   4 +-
 .../DefaultPartitionConditionBuilderTest.java   |  24 +-
 .../apache/kylin/storage/StorageMockUtils.java  |  33 +-
 .../storage/gtrecord/DictGridTableTest.java |   2 +-
 .../kylin/engine/mr/steps/CubeSamplingTest.java |   2 +-
 .../kylin/engine/mr/steps/MockupMapContext.java | 522 ++-
 .../mr/steps/NumberDictionaryForestTest.java|  18 +
 .../mr/steps/SelfDefineSortableKeyTest.java |  18 +
 .../apache/kylin/jdbc/SQLResonseStubTest.java   |   6 +-
 .../inmemcubing/ITDoggedCubeBuilderTest.java|  30 +-
 .../inmemcubing/ITInMemCubeBuilderTest.java |  30 +-
 pom.xml |   4 +-
 .../apache/kylin/query/udf/VersionUDFTest.java  |  18 +
 .../PasswordPlaceHolderConfigurerTest.java  |   1 -
 .../rest/controller/CubeControllerTest.java |   1 -
 .../config/KafkaConsumerPropertiesTest.java |  18 +
 .../hbase/common/HiveMiniClusterTest.java   | 131 -
 .../storage/hbase/steps/MockupMapContext.java   | 522 ++-
 45 files changed, 872 insertions(+), 883 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java 
b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 209986f..a426fc6 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -18,7 +18,11 @@
 
 package org.apache.kylin.common;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 import java.util.Map;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3e0edc84/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 9105245..bbaab09 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -82,7 +82,7 @@ public class BasicTest {
 
 @Test
 public void testxx() throws InterruptedException {
-System.out.println( 0x8fL);
+

[2/7] kylin git commit: KYLIN-2318 query cache is not working

[liyang]

KYLIN-2318 query cache is not working


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182c565d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182c565d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182c565d

Branch: refs/heads/master
Commit: 182c565df6b2134ef1302c88988ed27b60fefa42
Parents: fa4a5ee
Author: Hongbin Ma 
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../org/apache/kylin/common/QueryContext.java   | 61 
 .../kylin/common/debug/BackdoorToggles.java | 11 ++--
 .../apache/kylin/rest/request/SQLRequest.java   | 23 +---
 .../apache/kylin/rest/service/QueryService.java | 23 
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  4 +-
 5 files changed, 93 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java 
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for 
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new 
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.Pair;
 
 /**
+ * BackdoorToggles and QueryContext are similar because they're both hosting 
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work 
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ * 
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to 
SQLRequest
  */
 public class BackdoorToggles {
 
@@ -65,10 +70,6 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-}
-
 public static Pair getShardAssignment() {
 String v = 

[4/7] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log

[liyang]

introduce CheckUtil to make multiple condition checks clearer in log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2436140e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2436140e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2436140e

Branch: refs/heads/master
Commit: 2436140ef093c8d5b061124a5dbc2f5a1c3e9f3f
Parents: 182c565
Author: Hongbin Ma 
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../org/apache/kylin/common/util/CheckUtil.java | 33 
 .../apache/kylin/rest/service/QueryService.java | 17 ++
 2 files changed, 44 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message, 
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.rest.service;
 
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
 import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
 long startTime = System.currentTimeMillis();
 
 SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && 
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled = 
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in 
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query 
cache disabled in BackdoorToggles");
+
 if (queryCacheEnabled) {
 sqlResponse = searchQueryInCache(sqlRequest);
 }
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
 sqlResponse.setDuration(System.currentTimeMillis() - 
startTime);
 logger.info("Stats of SQL response: isException: {}, 
duration: {}, total scan count {}", //
 String.valueOf(sqlResponse.getIsException()), 
String.valueOf(sqlResponse.getDuration()), 
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold || 
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() < 
kylinConfig.getLargeQueryThreshold())) { //don't cache too large response

[3/7] kylin git commit: minor code refactors

[liyang]

minor code refactors


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa4a5ee2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa4a5ee2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa4a5ee2

Branch: refs/heads/master
Commit: fa4a5ee269eea66eee5b8ab8278305dfe0144c92
Parents: 00de441
Author: Hongbin Ma 
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 ++-
 .../org/apache/kylin/common/util/BasicTest.java |  1 +
 .../gtrecord/DummyPartitionStreamer.java| 40 
 .../storage/gtrecord/IPartitionStreamer.java| 26 +
 .../gtrecord/StorageResponseGTScatter.java  |  9 +++--
 .../apache/kylin/rest/service/QueryService.java |  6 ++-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  9 +++--
 7 files changed, 87 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8080577..d73b694 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -750,7 +750,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
 }
 
-@Deprecated//Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
+@Deprecated //Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }
@@ -759,6 +759,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan-threshold", 
"1000"));
 }
 
+public int getLargeQueryThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.large-query-threshold", 
String.valueOf((int) (getScanThreshold() * 0.1;
+}
+
 public int getDerivedInThreshold() {
 return 
Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold",
 "20"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 000..4caaed0
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language 

[5/7] kylin git commit: close OLAPEnumerator at exception

[liyang]

close OLAPEnumerator at exception

bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/596ab51b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/596ab51b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/596ab51b

Branch: refs/heads/master
Commit: 596ab51b4b98ae0923a134ede56c0ceb2f23bb66
Parents: 53b6c8c
Author: Hongbin Ma 
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:17:30 2016 +0800

--
 .../kylin/query/enumerator/OLAPEnumerator.java  | 33 +---
 1 file changed, 22 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/596ab51b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,31 @@ public class OLAPEnumerator implements Enumerator 
{
 
 @Override
 public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
 
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
 
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} catch (Exception e) {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
+}
+throw e;
 }
-convertCurrentRow(tuple);
-return true;
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {

[1/7] kylin git commit: strenthen storage visit deadline calculation

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 00de44116 -> 1563eb945


strenthen storage visit deadline calculation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53b6c8c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53b6c8c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53b6c8c1

Branch: refs/heads/master
Commit: 53b6c8c18033f92feadd2295456fa4f31011b5bb
Parents: 2436140
Author: Hongbin Ma 
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/53b6c8c1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 }
 
 final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() + 
scanReq.getTimeout();
 logger.info("deadline is " + deadline);
 final long storagePushDownLimit = 
scanReq.getStoragePushDownLimit();
 

[6/7] kylin git commit: add a toggle for dumpping storage partitions

[liyang]

add a toggle for dumpping storage partitions

add more log

temp for dumpped partitions

minor fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bfe8b1a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bfe8b1a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bfe8b1a3

Branch: refs/heads/master
Commit: bfe8b1a35c6b2c2f377eeb72295622ecb4d71284
Parents: 596ab51
Author: Hongbin Ma 
Authored: Fri Dec 23 13:56:01 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:17:33 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java | 30 
 .../gtrecord/StorageResponseGTScatter.java  |  2 ++
 2 files changed, 32 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index ca4a19c..8109cf2 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -62,6 +62,14 @@ public class BackdoorToggles {
 return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
 }
 
+public static String getPartitionDumpDir() {
+return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR);
+}
+
+public static String getDumpedPartitionDir() {
+return getString(DEBUG_TOGGLE_DUMPED_PARTITION_DIR);
+}
+
 public static int getQueryTimeout() {
 String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT);
 if (v == null)
@@ -190,6 +198,28 @@ public class BackdoorToggles {
  */
 public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = 
"DEBUG_TOGGLE_SHARD_ASSIGNMENT";
 
+/**
+ * set DEBUG_TOGGLE_PARTITION_DUMP_DIR="dir" to dump the partitions from 
storage.
+ * The dumped partitions are used for performance profiling, for example.
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_PARTITION_DUMP_DIR": "/tmp/dumping"
+ }
+ */
+public final static String DEBUG_TOGGLE_PARTITION_DUMP_DIR = 
"DEBUG_TOGGLE_PARTITION_DUMP_DIR";
+
+/**
+ * set DEBUG_TOGGLE_DUMPED_PARTITION_DIR="dir" to specify the dir to 
retrieve previously dumped partitions
+ * it's a companion toggle with DEBUG_TOGGLE_PARTITION_DUMP_DIR
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DUMPED_PARTITION_DIR": "/tmp/dumped"
+ }
+ */
+public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR = 
"DEBUG_TOGGLE_DUMPED_PARTITION_DIR";
+
 // properties on statement may go with this "channel" too
 /**
  * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property

http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 6283340..dc8746f 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -77,8 +77,10 @@ public class StorageResponseGTScatter implements IGTScanner {
 public Iterator iterator() {
 Iterator shardSubsets = 
Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
 if (storagePushDownLimit != Integer.MAX_VALUE) {
+logger.info("Using SortedIteratorMergerWithLimit to merge 
partitions");
 return new SortedIteratorMergerWithLimit(shardSubsets, 
storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
 } else {
+logger.info("Using Iterators.concat to merge partitions");
 return Iterators.concat(shardSubsets);
 }
 }

[04/12] kylin git commit: minor, drop scan threshold in KylinTestBase

[liyang]

minor, drop scan threshold in KylinTestBase


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00de4411
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00de4411
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00de4411

Branch: refs/heads/mhb-master
Commit: 00de44116126bd585f906c25606579a1ce2961f3
Parents: 74c2d6b
Author: Li Yang 
Authored: Tue Dec 27 18:38:16 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 27 18:38:16 2016 +0800

--
 kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/00de4411/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java 
b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 0a795ef..3ae7521 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -631,7 +631,6 @@ public class KylinTestBase {
 //setup cube conn
 File olapTmp = 
OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, 
config);
 Properties props = new Properties();
-props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "20001");
 cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + 
olapTmp.getAbsolutePath(), props);
 
 //setup h2

[09/12] kylin git commit: minor code refactors

[liyang]

minor code refactors


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa4a5ee2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa4a5ee2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa4a5ee2

Branch: refs/heads/mhb-master
Commit: fa4a5ee269eea66eee5b8ab8278305dfe0144c92
Parents: 00de441
Author: Hongbin Ma 
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 ++-
 .../org/apache/kylin/common/util/BasicTest.java |  1 +
 .../gtrecord/DummyPartitionStreamer.java| 40 
 .../storage/gtrecord/IPartitionStreamer.java| 26 +
 .../gtrecord/StorageResponseGTScatter.java  |  9 +++--
 .../apache/kylin/rest/service/QueryService.java |  6 ++-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  9 +++--
 7 files changed, 87 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8080577..d73b694 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -750,7 +750,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
 }
 
-@Deprecated//Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
+@Deprecated //Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }
@@ -759,6 +759,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan-threshold", 
"1000"));
 }
 
+public int getLargeQueryThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.large-query-threshold", 
String.valueOf((int) (getScanThreshold() * 0.1;
+}
+
 public int getDerivedInThreshold() {
 return 
Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold",
 "20"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 000..4caaed0
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language 

[05/12] kylin git commit: KYLIN-2318 query cache is not working

[liyang]

KYLIN-2318 query cache is not working


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182c565d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182c565d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182c565d

Branch: refs/heads/mhb-master
Commit: 182c565df6b2134ef1302c88988ed27b60fefa42
Parents: fa4a5ee
Author: Hongbin Ma 
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../org/apache/kylin/common/QueryContext.java   | 61 
 .../kylin/common/debug/BackdoorToggles.java | 11 ++--
 .../apache/kylin/rest/request/SQLRequest.java   | 23 +---
 .../apache/kylin/rest/service/QueryService.java | 23 
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  4 +-
 5 files changed, 93 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java 
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for 
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new 
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.Pair;
 
 /**
+ * BackdoorToggles and QueryContext are similar because they're both hosting 
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work 
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ * 
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to 
SQLRequest
  */
 public class BackdoorToggles {
 
@@ -65,10 +70,6 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-}
-
 public static Pair getShardAssignment() {
 String v = 

[02/12] kylin git commit: minor, fix UT

[liyang]

minor, fix UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9a8402d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9a8402d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9a8402d

Branch: refs/heads/mhb-master
Commit: f9a8402d57755a9c3c3132ff676cf3c7492775c9
Parents: a78b650
Author: Billy Liu 
Authored: Tue Dec 27 00:02:35 2016 +0800
Committer: Billy Liu 
Committed: Tue Dec 27 00:02:35 2016 +0800

--
 .../kylin/dict/DictionaryProviderTest.java  | 36 +---
 1 file changed, 32 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f9a8402d/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
 
b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
index a4aee76..4b386a7 100644
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
 package org.apache.kylin.dict;
 
 import static org.junit.Assert.assertTrue;
@@ -13,13 +31,23 @@ import java.util.Iterator;
 
 import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
-/**
- * Created by xiefan on 16-11-23.
- */
-public class DictionaryProviderTest {
+public class DictionaryProviderTest extends LocalFileMetadataTestCase{
+
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
 
 @Test
 public void testReadWrite() throws Exception {

[03/12] kylin git commit: minor, more stable DeployUtil

[liyang]

minor, more stable DeployUtil


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74c2d6bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74c2d6bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74c2d6bc

Branch: refs/heads/mhb-master
Commit: 74c2d6bcf0ea96939c63c3432961d666c77e609c
Parents: f9a8402
Author: Yang Li 
Authored: Tue Dec 27 07:02:32 2016 +0800
Committer: Yang Li 
Committed: Tue Dec 27 07:02:32 2016 +0800

--
 assembly/src/test/java/org/apache/kylin/job/DeployUtil.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/74c2d6bc/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 089c3ed..a42b125 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -255,8 +255,9 @@ public class DeployUtil {
 private static String[] generateCreateTableHql(TableDesc tableDesc) {
 
 String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity();
+String dropsql2 = "DROP VIEW IF EXISTS " + tableDesc.getIdentity();
+
 StringBuilder ddl = new StringBuilder();
-
 ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
 ddl.append("(" + "\n");
 
@@ -272,7 +273,7 @@ public class DeployUtil {
 ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
 ddl.append("STORED AS TEXTFILE");
 
-return new String[] { dropsql, ddl.toString() };
+return new String[] { dropsql, dropsql2, ddl.toString() };
 }
 
 private static String[] generateCreateViewHql(String viewName, String 
tableName) {

[08/12] kylin git commit: bug fix

[liyang]

bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9f2d1145
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9f2d1145
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9f2d1145

Branch: refs/heads/mhb-master
Commit: 9f2d1145640be3dd47b90fcb1b64d796f87b5fbd
Parents: e14aadf
Author: Hongbin Ma 
Authored: Sun Dec 25 14:51:08 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9f2d1145/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 408d227..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -76,14 +76,15 @@ public class OLAPEnumerator implements Enumerator 
{
 }
 convertCurrentRow(tuple);
 return true;
-} finally {
+} catch (Exception e) {
 try {
 if (cursor != null) {
 cursor.close();
 }
-} catch (Exception e) {
-logger.info("Error when closing cursor, ignore it", e);
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
 }
+throw e;
 }
 }
 

[01/12] kylin git commit: KYLIN-2322 make TrieDictionaryForest configurable, off by default [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/mhb-master b1f59c9fa -> 5c75380f0 (forced update)


KYLIN-2322 make TrieDictionaryForest configurable, off by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a78b6509
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a78b6509
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a78b6509

Branch: refs/heads/mhb-master
Commit: a78b6509f8727a53e26c57b2e5a14adc0ff88b73
Parents: 4f466a0
Author: Yang Li 
Authored: Mon Dec 26 21:12:20 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 26 21:14:01 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 ++
 .../apache/kylin/dict/DictionaryGenerator.java  | 64 ++--
 2 files changed, 63 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a78b6509/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index da93388..8080577 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,6 +216,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 // DICTIONARY & SNAPSHOT
 // 

 
+public boolean isUseForestTrieDictionary() {
+return 
Boolean.parseBoolean(getOptional("kylin.dictionary.use-forest-trie", "false"));
+}
+
 public int getTrieDictionaryForestMaxTrieSizeMB() {
 return 
Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a78b6509/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index cd13d59..6f4f2c4 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
@@ -49,10 +50,12 @@ public class DictionaryGenerator {
 builder = new DateDictBuilder();
 else
 builder = new TimeDictBuilder();
-} else if (dataType.isNumberFamily()) {
-builder = new NumberDictBuilder();
 } else {
-builder = new StringDictBuilder();
+boolean useForest = 
KylinConfig.getInstanceFromEnv().isUseForestTrieDictionary();
+if (dataType.isNumberFamily())
+builder = useForest ? new NumberTrieDictForestBuilder() : new 
NumberTrieDictBuilder();
+else
+builder = useForest ? new StringTrieDictForestBuilder() : new 
StringTrieDictBuilder();
 }
 return builder;
 }
@@ -168,7 +171,32 @@ public class DictionaryGenerator {
 }
 }
 
-private static class StringDictBuilder implements IDictionaryBuilder {
+private static class StringTrieDictBuilder implements IDictionaryBuilder {
+int baseId;
+TrieDictionaryBuilder builder;
+
+@Override
+public void init(DictionaryInfo info, int baseId) throws IOException {
+this.baseId = baseId;
+this.builder = new TrieDictionaryBuilder(new 
StringBytesConverter());
+}
+
+@Override
+public boolean addValue(String value) {
+if (value == null)
+return false;
+
+builder.addValue(value);
+return true;
+}
+
+@Override
+public Dictionary build() throws IOException {
+return builder.build(baseId);
+}
+}
+
+private static class StringTrieDictForestBuilder implements 
IDictionaryBuilder {
 TrieDictionaryForestBuilder builder;
 
 @Override
@@ -191,7 +219,33 @@ public class DictionaryGenerator {
 }
 }
 
-private static class NumberDictBuilder implements IDictionaryBuilder {
+@SuppressWarnings("deprecation")
+private static class 

[10/12] kylin git commit: KYLIN-2318 bug fix

[liyang]

KYLIN-2318 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c75380f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c75380f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c75380f

Branch: refs/heads/mhb-master
Commit: 5c75380f0d7bf873e07e7504ae4d407fa8a77f7c
Parents: 24db7c1
Author: Hongbin Ma 
Authored: Tue Dec 27 12:47:08 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../org/apache/kylin/common/debug/BackdoorToggles.java| 10 ++
 .../java/org/apache/kylin/rest/service/QueryService.java  |  1 +
 2 files changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5c75380f/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 8109cf2..95d5d62 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -23,6 +23,8 @@ import java.util.Map;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.Pair;
 
+import com.google.common.collect.Maps;
+
 /**
  * BackdoorToggles and QueryContext are similar because they're both hosting 
per-query thread local variables.
  * The difference is that BackdoorToggles are specified by user input and work 
for debug purpose. QueryContext
@@ -38,6 +40,14 @@ public class BackdoorToggles {
 _backdoorToggles.set(toggles);
 }
 
+public static void addToggles(Map toggles) {
+Map map = _backdoorToggles.get();
+if (map == null) {
+setToggles(Maps. newHashMap());
+}
+_backdoorToggles.get().putAll(toggles);
+}
+
 public static String getCoprocessorBehavior() {
 return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c75380f/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index bb0342f..de28b17 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,6 +327,7 @@ public class QueryService extends BasicService {
 }
 
 final String queryId = UUID.randomUUID().toString();
+BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
 QueryContext.setQueryId(queryId);
 
 try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {

[12/12] kylin git commit: add a toggle for dumpping storage partitions

[liyang]

add a toggle for dumpping storage partitions

add more log

temp for dumpped partitions

minor fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24db7c16
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24db7c16
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24db7c16

Branch: refs/heads/mhb-master
Commit: 24db7c167b5d8f6c1797169ae90d1d573e2c737c
Parents: 9f2d114
Author: Hongbin Ma 
Authored: Fri Dec 23 13:56:01 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java | 30 
 .../gtrecord/StorageResponseGTScatter.java  |  2 ++
 2 files changed, 32 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/24db7c16/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index ca4a19c..8109cf2 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -62,6 +62,14 @@ public class BackdoorToggles {
 return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
 }
 
+public static String getPartitionDumpDir() {
+return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR);
+}
+
+public static String getDumpedPartitionDir() {
+return getString(DEBUG_TOGGLE_DUMPED_PARTITION_DIR);
+}
+
 public static int getQueryTimeout() {
 String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT);
 if (v == null)
@@ -190,6 +198,28 @@ public class BackdoorToggles {
  */
 public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = 
"DEBUG_TOGGLE_SHARD_ASSIGNMENT";
 
+/**
+ * set DEBUG_TOGGLE_PARTITION_DUMP_DIR="dir" to dump the partitions from 
storage.
+ * The dumped partitions are used for performance profiling, for example.
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_PARTITION_DUMP_DIR": "/tmp/dumping"
+ }
+ */
+public final static String DEBUG_TOGGLE_PARTITION_DUMP_DIR = 
"DEBUG_TOGGLE_PARTITION_DUMP_DIR";
+
+/**
+ * set DEBUG_TOGGLE_DUMPED_PARTITION_DIR="dir" to specify the dir to 
retrieve previously dumped partitions
+ * it's a companion toggle with DEBUG_TOGGLE_PARTITION_DUMP_DIR
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DUMPED_PARTITION_DIR": "/tmp/dumped"
+ }
+ */
+public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR = 
"DEBUG_TOGGLE_DUMPED_PARTITION_DIR";
+
 // properties on statement may go with this "channel" too
 /**
  * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property

http://git-wip-us.apache.org/repos/asf/kylin/blob/24db7c16/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 6283340..dc8746f 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -77,8 +77,10 @@ public class StorageResponseGTScatter implements IGTScanner {
 public Iterator iterator() {
 Iterator shardSubsets = 
Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
 if (storagePushDownLimit != Integer.MAX_VALUE) {
+logger.info("Using SortedIteratorMergerWithLimit to merge 
partitions");
 return new SortedIteratorMergerWithLimit(shardSubsets, 
storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
 } else {
+logger.info("Using Iterators.concat to merge partitions");
 return Iterators.concat(shardSubsets);
 }
 }

[07/12] kylin git commit: close OLAPEnumerator at exception

[liyang]

close OLAPEnumerator at exception


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e14aadf6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e14aadf6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e14aadf6

Branch: refs/heads/mhb-master
Commit: e14aadf6a5d64dc47d4a7f26feaae56c8f80ff87
Parents: 53b6c8c
Author: Hongbin Ma 
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../kylin/query/enumerator/OLAPEnumerator.java  | 32 +---
 1 file changed, 21 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e14aadf6/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..408d227 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,30 @@ public class OLAPEnumerator implements Enumerator 
{
 
 @Override
 public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
 
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
 
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} finally {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception e) {
+logger.info("Error when closing cursor, ignore it", e);
+}
 }
-convertCurrentRow(tuple);
-return true;
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {

[06/12] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log

[liyang]

introduce CheckUtil to make multiple condition checks clearer in log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2436140e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2436140e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2436140e

Branch: refs/heads/mhb-master
Commit: 2436140ef093c8d5b061124a5dbc2f5a1c3e9f3f
Parents: 182c565
Author: Hongbin Ma 
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../org/apache/kylin/common/util/CheckUtil.java | 33 
 .../apache/kylin/rest/service/QueryService.java | 17 ++
 2 files changed, 44 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message, 
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.rest.service;
 
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
 import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
 long startTime = System.currentTimeMillis();
 
 SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && 
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled = 
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in 
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query 
cache disabled in BackdoorToggles");
+
 if (queryCacheEnabled) {
 sqlResponse = searchQueryInCache(sqlRequest);
 }
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
 sqlResponse.setDuration(System.currentTimeMillis() - 
startTime);
 logger.info("Stats of SQL response: isException: {}, 
duration: {}, total scan count {}", //
 String.valueOf(sqlResponse.getIsException()), 
String.valueOf(sqlResponse.getDuration()), 
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold || 
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() < 
kylinConfig.getLargeQueryThreshold())) { //don't cache too large 

[11/12] kylin git commit: strenthen storage visit deadline calculation

[liyang]

strenthen storage visit deadline calculation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53b6c8c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53b6c8c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53b6c8c1

Branch: refs/heads/mhb-master
Commit: 53b6c8c18033f92feadd2295456fa4f31011b5bb
Parents: 2436140
Author: Hongbin Ma 
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 27 22:15:44 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/53b6c8c1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 }
 
 final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() + 
scanReq.getTimeout();
 logger.info("deadline is " + deadline);
 final long storagePushDownLimit = 
scanReq.getStoragePushDownLimit();
 

kylin git commit: KYLIN-2317 Add HybridCubeCLI

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYILN-2317 [created] 48eaa3642


KYLIN-2317 Add HybridCubeCLI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48eaa364
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48eaa364
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48eaa364

Branch: refs/heads/KYILN-2317
Commit: 48eaa3642a7ca5771c3d8b7278682c501091ef41
Parents: 74c2d6b
Author: Billy Liu 
Authored: Tue Dec 27 18:03:06 2016 +0800
Committer: Billy Liu 
Committed: Tue Dec 27 18:03:06 2016 +0800

--
 .../localmeta/cube/ssb_cube1.json   |  14 ++
 .../localmeta/cube/ssb_cube2.json   |  14 ++
 .../localmeta/cube_desc/ssb_cube1.json  | 155 +
 .../localmeta/cube_desc/ssb_cube2.json  | 155 +
 .../org/apache/kylin/tool/HybridCubeCLI.java| 165 +++
 .../apache/kylin/tool/HybridCubeCLITest.java|  82 +
 6 files changed, 585 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/48eaa364/examples/test_case_data/localmeta/cube/ssb_cube1.json
--
diff --git a/examples/test_case_data/localmeta/cube/ssb_cube1.json 
b/examples/test_case_data/localmeta/cube/ssb_cube1.json
new file mode 100644
index 000..b1eb2a5
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ssb_cube1.json
@@ -0,0 +1,14 @@
+{
+  "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c70",
+  "name" : "ssb_cube1",
+  "owner" : "ADMIN",
+  "cost" : 50,
+  "status" : "DISABLED",
+  "segments" : [ ],
+  "last_modified" : 1457534216410,
+  "descriptor" : "ssb_cube1",
+  "create_time_utc" : 1457444500888,
+  "size_kb" : 0,
+  "input_records_count" : 0,
+  "input_records_size" : 0
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/48eaa364/examples/test_case_data/localmeta/cube/ssb_cube2.json
--
diff --git a/examples/test_case_data/localmeta/cube/ssb_cube2.json 
b/examples/test_case_data/localmeta/cube/ssb_cube2.json
new file mode 100644
index 000..58f935d
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ssb_cube2.json
@@ -0,0 +1,14 @@
+{
+  "uuid" : "70a9f288-3c01-4745-a04b-5641e82d6c71",
+  "name" : "ssb_cube2",
+  "owner" : "ADMIN",
+  "cost" : 50,
+  "status" : "DISABLED",
+  "segments" : [ ],
+  "last_modified" : 1457534216410,
+  "descriptor" : "ssb_cube2",
+  "create_time_utc" : 1457444500888,
+  "size_kb" : 0,
+  "input_records_count" : 0,
+  "input_records_size" : 0
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/48eaa364/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
--
diff --git a/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json 
b/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
new file mode 100644
index 000..5986057
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ssb_cube1.json
@@ -0,0 +1,155 @@
+{
+  "uuid" : "5c44df30-daec-486e-af90-927bf7851058",
+  "name" : "ssb_cube1",
+  "description" : "",
+  "dimensions" : [ {
+"name" : "SSB.PART_DERIVED",
+"table" : "SSB.PART",
+"column" : null,
+"derived" : [ "P_MFGR", "P_CATEGORY", "P_BRAND" ]
+  }, {
+"name" : "C_CITY",
+"table" : "SSB.CUSTOMER",
+"column" : "C_CITY",
+"derived" : null
+  }, {
+"name" : "C_REGION",
+"table" : "SSB.CUSTOMER",
+"column" : "C_REGION",
+"derived" : null
+  }, {
+"name" : "C_NATION",
+"table" : "SSB.CUSTOMER",
+"column" : "C_NATION",
+"derived" : null
+  }, {
+"name" : "D_YEAR",
+"table" : "SSB.DATES",
+"column" : "D_YEAR",
+"derived" : null
+  }, {
+"name" : "D_YEARMONTH",
+"table" : "SSB.DATES",
+"column" : "D_YEARMONTH",
+"derived" : null
+  }, {
+"name" : "D_YEARMONTHNUM",
+"table" : "SSB.DATES",
+"column" : "D_YEARMONTHNUM",
+"derived" : null
+  }, {
+"name" : "D_WEEKNUMINYEAR",
+"table" : "SSB.DATES",
+"column" : "D_WEEKNUMINYEAR",
+"derived" : null
+  } ],
+  "measures" : [ {
+"name" : "_COUNT_",
+"function" : {
+  "expression" : "COUNT",
+  "parameter" : {
+"type" : "constant",
+"value" : "1",
+"next_parameter" : null
+  },
+  "returntype" : "bigint"
+},
+"dependent_measure_ref" : null
+  }, {
+"name" : "TOTAL_REVENUE",
+"function" : {
+  "expression" : "SUM",
+  "parameter" : {
+"type" : "column",
+"value" : "LO_REVENUE",
+"next_parameter" : null
+  },
+  "returntype" : "bigint"
+},
+"dependent_measure_ref" : null
+  },