[2/5] kylin git commit: encoding add filter by column type

[mahongbin]

encoding add filter by column type

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/258353f5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/258353f5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/258353f5

Branch: refs/heads/yang21-hbase1.x
Commit: 258353f5718ce007313fd723b3ba62cb82c1d9f4
Parents: 577e6be
Author: luguosheng <550175...@qq.com>
Authored: Sat Nov 19 16:49:41 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 09:39:26 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 51 +++---
 webapp/app/js/controllers/cubeMeasures.js | 33 +
 webapp/app/js/utils/utils.js  | 21 ---
 3 files changed, 70 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/258353f5/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 5c5d9c7..3282974 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -77,33 +77,40 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   },function(e){
 $scope.store.supportedEncoding = $scope.cubeConfig.encodings;
   })
-
+  $scope.createFilter=function(type){
+ if(type.indexOf("varchar")<=0){
+   return ['fixed_length_hex'];
+ }else if(type!="date"){
+   return ['date'];
+ }else if(type!="time"&!="datetime"&!="timestamp"){
+   return ['time'];
+ }else{
+   return [];
+ }
+  }
   $scope.getEncodings =function (name){
-var type = TableModel.columnNameTypeMap[name]||'';
-var encodings =[];
-for(var k=0;k<$scope.store.supportedEncoding.length;k++){
-  encodings.push($scope.store.supportedEncoding[k]);
-}
+var filterName=name;
+var type = TableModel.columnNameTypeMap[filterName]||'';
+var encodings =$scope.store.supportedEncoding,filterEncoding;
+var filerList=$scope.createFilter(type);
 if($scope.isEdit){
-  for(var s=0;s<$scope.cubeMetaFrame.rowkey.rowkey_columns.length;s++){
-if(name==$scope.cubeMetaFrame.rowkey.rowkey_columns[s].column){
-  var 
version=$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding_version;
-  
encodings=VdmUtil.changeObjectListValueByFilter(encodings,'value',$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding.replace(/:\d+/,"")+(version?"[v"+version+"]":"[v1]"),'suggest',true)
+  if($scope.cubeMetaFrame.rowkey.rowkey_columns&){
+for(var s=0;s<$scope.cubeMetaFrame.rowkey.rowkey_columns.length;s++){
+  if(filterName==$scope.cubeMetaFrame.rowkey.rowkey_columns[s].column){
+var 
version=$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding_version;
+
filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'value',$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding.replace(/:\d+/,"")+(version?"[v"+version+"]":"[v1]"),'suggest',true)
+  }
 }
+  }else{
+
filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'suggest',true);
   }
+}else{
+  
filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'suggest',true);
+}
+for(var f=0;f

[1/5] kylin git commit: encoding at version tips [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 73ec6fe13 -> edb9d54e9 (forced update)


encoding at version tips

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/577e6be2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/577e6be2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/577e6be2

Branch: refs/heads/yang21-hbase1.x
Commit: 577e6be2d08faf821061d037df8f1576ec2c5d57
Parents: 61dd19f
Author: luguosheng <550175...@qq.com>
Authored: Sat Nov 19 14:50:52 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 09:38:59 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js |  68 -
 webapp/app/js/controllers/cubeEdit.js   |  90 ++---
 webapp/app/js/controllers/cubeMeasures.js   | 101 +--
 webapp/app/js/services/cubes.js |   2 +-
 webapp/app/js/utils/utils.js|  45 +
 5 files changed, 211 insertions(+), 95 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/577e6be2/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index be90f65..f2b36fe 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -19,34 +19,46 @@
 'use strict';
 
 KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, 
$modal,cubeConfig,MetaModel,cubesManager,CubeDescModel,SweetAlert) {
-$scope.cubesManager = cubesManager;
-
-
+  $scope.cubesManager = cubesManager;
+  $scope.getTypeVersion=function(typename){
+var searchResult=/\[v(\d+)\]/.exec(typename);
+if(searchResult&){
+  return searchResult.length&[1]||1;
+}else{
+  return 1;
+}
+  }
+  $scope.removeVersion=function(typename){
+if(typename){
+  return typename.replace(/\[v\d+\]/g,"");
+}
+return "";
+  }
+  var needLengthKeyList=['fixed_length','fixed_length_hex','int','integer'];
   //rowkey
   $scope.convertedRowkeys = [];
   angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(item){
 //var _isDictionaries = item.encoding === "dict"?"true":"false";
-var _isFixedLength = item.encoding.substring(0,12) === 
"fixed_length"?"true":"false";//fixed_length:12
-var _isIntLength = item.encoding.substring(0,3) === 
"int"?"true":"false";//fixed_length:12
+//var version=$scope.getTypeVersion(encoding);
+item.encoding=$scope.removeVersion(item.encoding);
+//var _isFixedLength = item.encoding.substring(0,12) === 
"fixed_length"?"true":"false";//fixed_length:12
+//var _isIntegerLength = item.encoding.substring(0,7) === 
"integer"?"true":"false";
+//var _isIntLength = item.encoding.substring(0,3) === "int"?"true":"false";
 var _encoding = item.encoding;
 var _valueLength ;
-if(_isFixedLength !=="false"){
-  _valueLength = item.encoding.substring(13,item.encoding.length);
-  _encoding = "fixed_length";
-}
-if(_isIntLength!="false"){
-  _valueLength = item.encoding.substring(4,item.encoding.length);
-  _encoding = "int";
+var baseKey=item.encoding.replace(/:\d+/,'');
+if(needLengthKeyList.indexOf(baseKey)>=-1){
+  var result=/:(\d+)/.exec(item.encoding);
+  _valueLength=result?result[1]:0;
 }
-
+_encoding=baseKey;
 var rowkeyObj = {
   column:item.column,
-  encoding:_encoding,
+  
encoding:_encoding+(item.encoding_version?"[v"+item.encoding_version+"]":"[v1]"),
   valueLength:_valueLength,
-  isShardBy:item.isShardBy
-
+  isShardBy:item.isShardBy,
+  encoding_version:item.encoding_version||1
 }
-
 $scope.convertedRowkeys.push(rowkeyObj);
 
   })
@@ -55,26 +67,26 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
   $scope.rule={
 shardColumnAvailable:true
   }
-
+  var 
checkedlen=$scope.cubeMetaFrame.rowkey.rowkey_columns&&$scope.cubeMetaFrame.rowkey.rowkey_columns.length||0;
+  for(var i=0;i

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/edb9d54e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/edb9d54e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/3] kylin git commit: encoding at version tips

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21 61dd19f6f -> d4715b559


encoding at version tips

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/577e6be2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/577e6be2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/577e6be2

Branch: refs/heads/yang21
Commit: 577e6be2d08faf821061d037df8f1576ec2c5d57
Parents: 61dd19f
Author: luguosheng <550175...@qq.com>
Authored: Sat Nov 19 14:50:52 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 09:38:59 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js |  68 -
 webapp/app/js/controllers/cubeEdit.js   |  90 ++---
 webapp/app/js/controllers/cubeMeasures.js   | 101 +--
 webapp/app/js/services/cubes.js |   2 +-
 webapp/app/js/utils/utils.js|  45 +
 5 files changed, 211 insertions(+), 95 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/577e6be2/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index be90f65..f2b36fe 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -19,34 +19,46 @@
 'use strict';
 
 KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, 
$modal,cubeConfig,MetaModel,cubesManager,CubeDescModel,SweetAlert) {
-$scope.cubesManager = cubesManager;
-
-
+  $scope.cubesManager = cubesManager;
+  $scope.getTypeVersion=function(typename){
+var searchResult=/\[v(\d+)\]/.exec(typename);
+if(searchResult&){
+  return searchResult.length&[1]||1;
+}else{
+  return 1;
+}
+  }
+  $scope.removeVersion=function(typename){
+if(typename){
+  return typename.replace(/\[v\d+\]/g,"");
+}
+return "";
+  }
+  var needLengthKeyList=['fixed_length','fixed_length_hex','int','integer'];
   //rowkey
   $scope.convertedRowkeys = [];
   angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(item){
 //var _isDictionaries = item.encoding === "dict"?"true":"false";
-var _isFixedLength = item.encoding.substring(0,12) === 
"fixed_length"?"true":"false";//fixed_length:12
-var _isIntLength = item.encoding.substring(0,3) === 
"int"?"true":"false";//fixed_length:12
+//var version=$scope.getTypeVersion(encoding);
+item.encoding=$scope.removeVersion(item.encoding);
+//var _isFixedLength = item.encoding.substring(0,12) === 
"fixed_length"?"true":"false";//fixed_length:12
+//var _isIntegerLength = item.encoding.substring(0,7) === 
"integer"?"true":"false";
+//var _isIntLength = item.encoding.substring(0,3) === "int"?"true":"false";
 var _encoding = item.encoding;
 var _valueLength ;
-if(_isFixedLength !=="false"){
-  _valueLength = item.encoding.substring(13,item.encoding.length);
-  _encoding = "fixed_length";
-}
-if(_isIntLength!="false"){
-  _valueLength = item.encoding.substring(4,item.encoding.length);
-  _encoding = "int";
+var baseKey=item.encoding.replace(/:\d+/,'');
+if(needLengthKeyList.indexOf(baseKey)>=-1){
+  var result=/:(\d+)/.exec(item.encoding);
+  _valueLength=result?result[1]:0;
 }
-
+_encoding=baseKey;
 var rowkeyObj = {
   column:item.column,
-  encoding:_encoding,
+  
encoding:_encoding+(item.encoding_version?"[v"+item.encoding_version+"]":"[v1]"),
   valueLength:_valueLength,
-  isShardBy:item.isShardBy
-
+  isShardBy:item.isShardBy,
+  encoding_version:item.encoding_version||1
 }
-
 $scope.convertedRowkeys.push(rowkeyObj);
 
   })
@@ -55,26 +67,26 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
   $scope.rule={
 shardColumnAvailable:true
   }
-
+  var 
checkedlen=$scope.cubeMetaFrame.rowkey.rowkey_columns&&$scope.cubeMetaFrame.rowkey.rowkey_columns.length||0;
+  for(var i=0;i

[3/3] kylin git commit: add utilsjs link

[mahongbin]

add utilsjs link

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4715b55
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4715b55
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4715b55

Branch: refs/heads/yang21
Commit: d4715b559dda73457610f41374fc0042ffdbddf2
Parents: 258353f
Author: luguosheng <550175...@qq.com>
Authored: Sat Nov 19 19:44:53 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 09:39:42 2016 +0800

--
 webapp/app/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d4715b55/webapp/app/index.html
--
diff --git a/webapp/app/index.html b/webapp/app/index.html
index 8fb2cb9..ad881d5 100644
--- a/webapp/app/index.html
+++ b/webapp/app/index.html
@@ -163,7 +163,7 @@
 
 
 
-
+
 
 
 

[2/3] kylin git commit: encoding add filter by column type

[mahongbin]

encoding add filter by column type

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/258353f5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/258353f5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/258353f5

Branch: refs/heads/yang21
Commit: 258353f5718ce007313fd723b3ba62cb82c1d9f4
Parents: 577e6be
Author: luguosheng <550175...@qq.com>
Authored: Sat Nov 19 16:49:41 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 09:39:26 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 51 +++---
 webapp/app/js/controllers/cubeMeasures.js | 33 +
 webapp/app/js/utils/utils.js  | 21 ---
 3 files changed, 70 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/258353f5/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 5c5d9c7..3282974 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -77,33 +77,40 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   },function(e){
 $scope.store.supportedEncoding = $scope.cubeConfig.encodings;
   })
-
+  $scope.createFilter=function(type){
+ if(type.indexOf("varchar")<=0){
+   return ['fixed_length_hex'];
+ }else if(type!="date"){
+   return ['date'];
+ }else if(type!="time"&!="datetime"&!="timestamp"){
+   return ['time'];
+ }else{
+   return [];
+ }
+  }
   $scope.getEncodings =function (name){
-var type = TableModel.columnNameTypeMap[name]||'';
-var encodings =[];
-for(var k=0;k<$scope.store.supportedEncoding.length;k++){
-  encodings.push($scope.store.supportedEncoding[k]);
-}
+var filterName=name;
+var type = TableModel.columnNameTypeMap[filterName]||'';
+var encodings =$scope.store.supportedEncoding,filterEncoding;
+var filerList=$scope.createFilter(type);
 if($scope.isEdit){
-  for(var s=0;s<$scope.cubeMetaFrame.rowkey.rowkey_columns.length;s++){
-if(name==$scope.cubeMetaFrame.rowkey.rowkey_columns[s].column){
-  var 
version=$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding_version;
-  
encodings=VdmUtil.changeObjectListValueByFilter(encodings,'value',$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding.replace(/:\d+/,"")+(version?"[v"+version+"]":"[v1]"),'suggest',true)
+  if($scope.cubeMetaFrame.rowkey.rowkey_columns&){
+for(var s=0;s<$scope.cubeMetaFrame.rowkey.rowkey_columns.length;s++){
+  if(filterName==$scope.cubeMetaFrame.rowkey.rowkey_columns[s].column){
+var 
version=$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding_version;
+
filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'value',$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding.replace(/:\d+/,"")+(version?"[v"+version+"]":"[v1]"),'suggest',true)
+  }
 }
+  }else{
+
filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'suggest',true);
   }
+}else{
+  
filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'suggest',true);
+}
+for(var f=0;f

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73ec6fe1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73ec6fe1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73ec6fe1

Branch: refs/heads/yang21-cdh5.7
Commit: 73ec6fe13c79ea216f8295004242f5cc956d969d
Parents: 61dd19f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 21 17:28:45 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec6fe1/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
- 

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 2107dc55f -> 2e166ad79 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec6fe1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec6fe1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-

[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e166ad7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e166ad7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e166ad7

Branch: refs/heads/yang21-cdh5.7
Commit: 2e166ad795009da99d3d54f93b9a0a660a35269f
Parents: 73ec6fe
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Mon Nov 21 17:30:37 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e166ad7/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e166ad7/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e166ad7/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e166ad7/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[2/3] kylin git commit: KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep

[mahongbin]

KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ab966b6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ab966b6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ab966b6

Branch: refs/heads/master
Commit: 3ab966b650debb83eb219c9ed8d357d141466776
Parents: a179f5e
Author: Hongbin Ma 
Authored: Fri Nov 18 14:56:30 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 21 13:37:58 2016 +0800

--
 .../kylin/engine/mr/common/CubeStatsReader.java|  2 +-
 .../kylin/engine/mr/steps/SaveStatisticsStep.java  | 17 ++---
 2 files changed, 15 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ab966b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c917cfb..c6839d6 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -227,7 +227,7 @@ public class CubeStatsReader {
 return ret;
 }
 
-private void print(PrintWriter out) {
+public void print(PrintWriter out) {
 Map cuboidRows = getCuboidRowEstimatesHLL();
 Map cuboidSizes = getCuboidSizeMap();
 List cuboids = new ArrayList(cuboidRows.keySet());

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ab966b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 23e81bc..7718bfb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,6 +19,8 @@
 package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.Random;
 
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -29,8 +31,8 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
+import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.engine.mr.common.CubeStatsReader;
 import org.apache.kylin.job.exception.ExecuteException;
@@ -85,6 +87,16 @@ public class SaveStatisticsStep extends AbstractExecutable {
 
 private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) 
throws IOException {
 String algPref = kylinConf.getCubeAlgorithm();
+
+CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
+StringWriter sw = new StringWriter();
+PrintWriter pw = new PrintWriter(sw);
+cubeStats.print(pw);
+pw.flush();
+pw.close();
+logger.info("Cube Stats Estimation for segment {} :", seg.toString());
+logger.info(sw.toString());
+
 AlgorithmEnum alg;
 if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
 alg = AlgorithmEnum.INMEM;
@@ -104,14 +116,13 @@ public class SaveStatisticsStep extends 
AbstractExecutable {
 } else if ("random".equalsIgnoreCase(algPref)) { // for testing
 alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : 
AlgorithmEnum.LAYER;
 } else { // the default
-CubeStatsReader cubeStats = new CubeStatsReader(seg, 
kylinConf);
 int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
 int mapperNumLimit = 
kylinConf.getCubeAlgorithmAutoMapperLimit();
 double mapperOverlapRatio = 
cubeStats.getMapperOverlapRatioOfFirstBuild();
 double overlapThreshold = 
kylinConf.getCubeAlgorithmAutoThreshold();
 logger.info("mapperNumber for " + seg + " is " + mapperNumber 
+ " and threshold is " + mapperNumLimit);
 logger.info("mapperOverlapRatio for " + seg + " is " + 
mapperOverlapRatio + " and threshold is " + overlapThreshold);
- 
+
 // in-mem cubing is good when

[1/3] kylin git commit: minor refactors to IGTCodeSystem

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master a179f5e83 -> d38d2e955


minor refactors to IGTCodeSystem


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b525d07
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b525d07
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b525d07

Branch: refs/heads/master
Commit: 4b525d074d82bccb89b0dad0cfa75f198ad3c4f0
Parents: 3ab966b
Author: Hongbin Ma 
Authored: Sat Nov 19 22:10:51 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 21 13:37:58 2016 +0800

--
 .../cube/gridtable/TrimmedCubeCodeSystem.java   |  8 ---
 .../java/org/apache/kylin/gridtable/GTInfo.java | 24 
 .../kylin/gridtable/GTSampleCodeSystem.java |  7 +++---
 3 files changed, 24 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4b525d07/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index 86c44e9..a0b230e 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.util.BytesSerializer;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.dimension.DictionaryDimEnc;
 import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.gridtable.IGTCodeSystem;
 import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 
 import com.google.common.collect.Maps;
@@ -87,9 +88,10 @@ public class TrimmedCubeCodeSystem extends CubeCodeSystem {
 }
 }
 
-public static final BytesSerializer serializer = 
new BytesSerializer() {
+public static final BytesSerializer serializer = new 
BytesSerializer() {
 @Override
-public void serialize(TrimmedCubeCodeSystem value, ByteBuffer out) {
+public void serialize(IGTCodeSystem ivalue, ByteBuffer out) {
+TrimmedCubeCodeSystem value = (TrimmedCubeCodeSystem) ivalue;
 BytesUtil.writeVInt(value.dependentMetricsMap.size(), out);
 for (Map.Entry x : 
value.dependentMetricsMap.entrySet()) {
 BytesUtil.writeVInt(x.getKey(), out);
@@ -105,7 +107,7 @@ public class TrimmedCubeCodeSystem extends CubeCodeSystem {
 }
 
 @Override
-public TrimmedCubeCodeSystem deserialize(ByteBuffer in) {
+public IGTCodeSystem deserialize(ByteBuffer in) {
 Map dependentMetricsMap = Maps.newHashMap();
 
 int size = BytesUtil.readVInt(in);

http://git-wip-us.apache.org/repos/asf/kylin/blob/4b525d07/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index 12a3593..ba62af3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -290,11 +290,17 @@ public class GTInfo {
 
BytesUtil.writeAsciiString(CubeCodeSystem.class.getCanonicalName(), out);
 TrimmedCubeCodeSystem trimmed = ((CubeCodeSystem) 
value.codeSystem).trimForCoprocessor();
 TrimmedCubeCodeSystem.serializer.serialize(trimmed, out);
-} else if (value.codeSystem instanceof GTSampleCodeSystem) {
-
BytesUtil.writeAsciiString(GTSampleCodeSystem.class.getCanonicalName(), out);
-GTSampleCodeSystem.serializer.serialize((GTSampleCodeSystem) 
value.codeSystem, out);
-} else {
+} else if (value.codeSystem != null) {
 
BytesUtil.writeAsciiString(value.codeSystem.getClass().getCanonicalName(), out);
+BytesSerializer serializer = null;
+try {
+serializer = (BytesSerializer) 
value.codeSystem.getClass().getField("serializer").get(null);
+} catch (IllegalAccessException | NoSuchFieldException e) {
+throw new RuntimeException("failed to get serializer for " 
+ value.codeSystem.getClass(), e);
+}
+serializer.serialize(value.codeSystem, out);
+} else {
+throw new IllegalStateException("code system cannot be null");
 }
 
 

[04/12] kylin git commit: KYLIN-2198 Add a framework to allow major changes in DimensionEncoding

[mahongbin]

KYLIN-2198 Add a framework to allow major changes in DimensionEncoding


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1acc419
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1acc419
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1acc419

Branch: refs/heads/yang21-cdh5.7
Commit: e1acc4192a982f897489f52d1bbc836a5c207da6
Parents: c0c0814
Author: Hongbin Ma 
Authored: Wed Nov 16 14:47:53 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:11:34 2016 +0800

--
 .../apache/kylin/common/util/JacksonBean.java   | 55 +++
 .../apache/kylin/common/util/JacksonTest.java   | 39 
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |  2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  | 13 ++-
 .../dimension/DimensionEncodingFactory.java | 97 +++-
 .../kylin/measure/topn/TopNMeasureType.java | 12 ++-
 .../kylin/rest/controller/CubeController.java   | 12 +--
 7 files changed, 198 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
new file mode 100644
index 000..42357f2
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
+public class JacksonBean {
+
+@JsonProperty("a")
+private String a;
+@JsonProperty("b")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+private int b;
+
+public String getA() {
+return a;
+}
+
+public void setA(String a) {
+this.a = a;
+}
+
+public int getB() {
+return b;
+}
+
+public void setB(int b) {
+this.b = b;
+}
+
+@Override
+public String toString() {
+return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
new file mode 100644
index 000..81be7eb
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+
+import org.junit.Test;
+

[06/12] kylin git commit: KYLIN-2201 fix combination check overflow error

[mahongbin]

KYLIN-2201 fix combination check overflow error

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9bd6a2ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9bd6a2ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9bd6a2ab

Branch: refs/heads/yang21-cdh5.7
Commit: 9bd6a2ab6b5fd8f11a80198012a519bfe4edac08
Parents: 02e723a
Author: Roger Shi 
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   |   6 +-
 .../kylin/cube/AggregationGroupRuleTest.java|  11 +
 .../org/apache/kylin/cube/CubeDescTest.java |   7 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 +++
 4 files changed, 419 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..c914ebb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -580,7 +580,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 throw new IllegalStateException("Aggregation group " + index + 
" select rule field not set");
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(includeDims, agg.getIncludes());
 
@@ -598,7 +598,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set jointDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
 if (jointDimsList.size() > 0) {
-combination = combination * (1 << jointDimsList.size());
+combination = combination * (1L << jointDimsList.size());
 }
 
 if (!includeDims.containsAll(mandatoryDims) || 
!includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) 
{
@@ -612,7 +612,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (combination > config.getCubeAggrGroupMaxCombination()) {
 String msg = "Aggregation group " + index + " has too many 
combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 
'kylin.cube.aggrgroup.max.combination' to a bigger value.";

http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 6c3d544..7fea440 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -115,6 +116,16 @@ public class AggregationGroupRuleTest {
 assertEquals("Aggregation group 0 joint columns overlap with more than 
1 dim in same hierarchy", (vContext.getResults()[0].getMessage()));
 }
 
+@Test
+public void testCombinationIntOverflow() throws IOException {
+ValidateContext vContext = new ValidateContext();
+CubeDesc desc = JsonUtil.readValue(new 
FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
+
+IValidatorRule rule = getAggregationGroupRule();
+rule.validate(desc, vContext);
+assertEquals(1, vContext.getResults().length);
+}
+
 public AggregationGroupRule getAggregationGroupRule() {
 AggregationGroupRule rule = new AggregationGroupRule() {
 @Override

[11/12] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a07a5292
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a07a5292
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a07a5292

Branch: refs/heads/yang21-cdh5.7
Commit: a07a5292b6352940cf0e5993890bc0b057ce3c3d
Parents: 61dd19f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Nov 19 22:17:14 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
- 

[05/12] kylin git commit: KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198

[mahongbin]

KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac4e9ec5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac4e9ec5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac4e9ec5

Branch: refs/heads/yang21-cdh5.7
Commit: ac4e9ec563b6e14e387c5803d441c8dbcbc06f87
Parents: e1acc41
Author: Hongbin Ma 
Authored: Wed Nov 16 14:48:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:14:24 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |   4 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   9 +-
 .../apache/kylin/dimension/IntegerDimEncV2.java | 228 +++
 .../apache/kylin/dimension/IntDimEncTest.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  |  18 +-
 5 files changed, 248 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 87d37be..d572e56 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.IntegerDimEncV2;
 import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -99,7 +99,7 @@ public class DimEncodingPreserveOrderTest {
 @Test
 public void testVLongDimEncPreserveOrder() {
 for (int i = 1; i <= successValue.size(); i++) {
-IntegerDimEnc enc = new IntegerDimEnc(i);
+IntegerDimEncV2 enc = new IntegerDimEncV2(i);
 List encodedValues = Lists.newArrayList();
 for (long value : successValue.get(i - 1)) {
 encodedValues.add(encode(enc, value));

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index e55a0a8..983af9a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -31,8 +31,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * replacement for IntegerDimEnc, the diff is VLongDimEnc supports negative 
values
+ * replacement for IntDimEnc, the diff is IntegerDimEnc supports negative 
values
+ * for IntegerDimEnc(N), the supported range is (-2^(8*N-1),2^(8*N-1))
+ *
+ * -2^(8*N-1) is not supported because the slot is reserved for null values.
+ * -2^(8*N-1) will be encoded with warn, and its output will be null
  */
+@Deprecated//due to a fatal bug (KYLIN-2191)
 public class IntegerDimEnc extends DimensionEncoding {
 private static final long serialVersionUID = 1L;
 
@@ -127,7 +132,7 @@ public class IntegerDimEnc extends DimensionEncoding {
 
 //only take useful bytes
 integer = integer & MASK[fixedLen];
-boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
+boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0;
 if (!positive) {
 integer |= (~MASK[fixedLen]);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
new file mode 100644
index 000..1a54664
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under 

[09/12] kylin git commit: minor, fix UT

[mahongbin]

minor, fix UT

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61dd19f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61dd19f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61dd19f6

Branch: refs/heads/yang21-cdh5.7
Commit: 61dd19f6f64e2fdcb3b09de5f4b1c171d942ed1c
Parents: d87b7a5
Author: Roger Shi 
Authored: Thu Nov 17 17:08:17 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 17 17:54:45 2016 +0800

--
 .../common/util/LocalFileMetadataTestCase.java  |   1 +
 .../kylin/cube/AggregationGroupRuleTest.java|  37 +-
 .../org/apache/kylin/cube/CubeDescTest.java |  13 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |   7 +
 .../validation/rule/DictionaryRuleTest.java |   3 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 -
 ..._cube_desc_combination_int_overflow.json.bad | 398 +
 .../cube_desc/ut_large_dimension_number.json| 585 ---
 .../ut_large_dimension_number.json.bad  | 585 +++
 9 files changed, 1036 insertions(+), 991 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61dd19f6/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index c3e2538..d351041 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
 
 public static String LOCALMETA_TEST_DATA = 
"../examples/test_case_data/localmeta";
+public static String LOCALMETA_TEMP_DATA = "../examples/test_metadata/";
 
 @Override
 public void createTestMetadata() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/61dd19f6/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 7fea440..7903017 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -33,15 +33,31 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.validation.IValidatorRule;
 import org.apache.kylin.cube.model.validation.ValidateContext;
 import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
+import org.apache.kylin.metadata.MetadataManager;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
-public class AggregationGroupRuleTest {
+public class AggregationGroupRuleTest extends LocalFileMetadataTestCase{
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+MetadataManager.clearCache();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
 
 @Test
 public void testGoodDesc() throws IOException {
 AggregationGroupRule rule = getAggregationGroupRule();
 
-for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/").listFiles()) {
+for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + 
"/cube_desc/").listFiles()) {
+if (!f.getName().endsWith("json")) {
+continue;
+}
 CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), 
CubeDesc.class);
 ValidateContext vContext = new ValidateContext();
 rule.validate(desc, vContext);
@@ -59,7 +75,7 @@ public class AggregationGroupRuleTest {
 }
 };
 
-for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/").listFiles()) {
+for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + 
"/cube_desc/").listFiles()) {
 System.out.println(f.getName());
 CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), 
CubeDesc.class);
 ValidateContext vContext = new ValidateContext();
@@ -74,7 +90,7 @@ public class AggregationGroupRuleTest {
 public void testGoodDesc2() throws IOException {
 
 ValidateContext vContext = new 

[02/12] kylin git commit: KYLIN-2191 fix integer encoding and refine UT

[mahongbin]

KYLIN-2191 fix integer encoding and refine UT

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3e7eb75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3e7eb75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3e7eb75

Branch: refs/heads/yang21-cdh5.7
Commit: b3e7eb751005b79c59958230832921ed4763e019
Parents: 2e9a59d
Author: Roger Shi 
Authored: Tue Nov 15 20:59:15 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:09 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |  85 +++-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  | 128 +++
 3 files changed, 129 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b3e7eb75/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index ffd43e5..87d37be 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -20,22 +20,68 @@ package org.apache.kylin.gridtable;
 
 import static org.junit.Assert.assertTrue;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Ordering;
 
 public class DimEncodingPreserveOrderTest {
+private static List successValue;
+private static List failValue;
+
+@BeforeClass
+public static void initTestValue() {
+successValue = new ArrayList<>();
+successValue.add(new long[] { -127, 0, 127 });
+successValue.add(new long[] { -32767, -127, 0, 127, 32767 });
+successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 
8388607 });
+successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 
127, 32767, 8388607, 2147483647L });
+successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, 
-32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L });
+successValue.add(new long[] { -140737488355327L, -549755813887L, 
-2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 
549755813887L, 140737488355327L });
+successValue.add(new long[] { -36028797018963967L, -140737488355327L, 
-549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 
2147483647L, 549755813887L, 140737488355327L, 36028797018963967L });
+successValue.add(new long[] { //
+-9223372036854775807L, //
+-36028797018963967L, //
+-140737488355327L, //
+-549755813887L, //
+-2147483647L, //
+-8388607, //
+-32767, //
+-127, //
+0, //
+127, // (2 ^ 7) - 1
+32767, // (2 ^ 15)  - 1
+8388607, // (2 ^ 23) - 1
+2147483647L, // (2 ^ 31) - 1
+549755813887L, // (2 ^ 39) - 1
+140737488355327L, // (2 ^ 47) - 1
+36028797018963967L, // (2 ^ 55) - 1
+9223372036854775807L }); // (2 ^ 63) - 1
+
+failValue = new ArrayList<>();
+failValue.add(new long[] { -128, 128 });
+failValue.add(new long[] { -32768, 32768 });
+failValue.add(new long[] { -8388608, 8388608 });
+failValue.add(new long[] { -2147483648L, 2147483648L });
+failValue.add(new long[] { -549755813888L, 549755813888L });
+failValue.add(new long[] { -140737488355328L, 140737488355328L });
+failValue.add(new long[] { -36028797018963968L, 36028797018963968L });
+failValue.add(new long[] { -9223372036854775808L });
+}
+
 @Test
 public void testOneMoreByteVLongDimEncPreserveOrder() {
+// TODO: better test
 OneMoreByteVLongDimEnc enc = new OneMoreByteVLongDimEnc(2);
 List encodedValues = Lists.newArrayList();
 

[08/12] kylin git commit: KYLIN-2199 bug fix

[mahongbin]

KYLIN-2199 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d87b7a51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d87b7a51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d87b7a51

Branch: refs/heads/yang21-cdh5.7
Commit: d87b7a51282ffb2e678af65c20b5a395a2347d49
Parents: 9bd6a2a
Author: Hongbin Ma 
Authored: Thu Nov 17 16:46:59 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 16:48:53 2016 +0800

--
 .../org/apache/kylin/dimension/DimensionEncodingFactory.java   | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d87b7a51/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 242e003..b617a19 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -73,6 +73,12 @@ public abstract class DimensionEncodingFactory {
 
 Map result = Maps.newHashMap();
 for (Pair p : factoryMap.keySet()) {
+if (result.containsKey(p.getFirst())) {
+if (result.get(p.getFirst()) > p.getSecond()) {
+continue;//skip small versions
+}
+}
+
 result.put(p.getFirst(), p.getSecond());
 }
 result.put(DictionaryDimEnc.ENCODING_NAME, 1);

[07/12] kylin git commit: KYLIN-2197 fix gg group buildMandatoryColumnMask

[mahongbin]

KYLIN-2197 fix gg group buildMandatoryColumnMask

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/02e723ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/02e723ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/02e723ae

Branch: refs/heads/yang21-cdh5.7
Commit: 02e723ae0c40847517c8e8c195cb723a5d70d275
Parents: ac4e9ec
Author: Li Yang 
Authored: Wed Nov 16 13:33:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  13 +
 .../cube_desc/ut_large_dimension_number.json| 585 +++
 .../model_desc/ut_large_dimension_number.json   |  26 +
 .../localmeta/table/DEFAULT.WIDE_TABLE.json | 217 +++
 5 files changed, 842 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 5f780d0..c7924c7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -124,7 +124,7 @@ public class AggregationGroup {
 for (String dim : mandatory_dims) {
 TblColRef hColumn = colNameAbbr.get(dim);
 Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
-mandatoryColumnMask |= 1 << index;
+mandatoryColumnMask |= (1L << index);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index ecb1200..bdceb0f 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -283,6 +284,18 @@ public class CuboidSchedulerTest extends 
LocalFileMetadataTestCase {
 assertEquals(cuboidScheduler.getCuboidCount(), sum);
 }
 
+@Test
+public void testCuboid_onlyBaseCuboid() {
+CubeDesc cube = 
getCubeDescManager().getCubeDesc("ut_large_dimension_number");
+CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
+assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+
+List spanningChild = 
scheduler.getSpanningCuboid(baseCuboid.getId());
+assertTrue(spanningChild.size() > 0);
+}
+
 public CubeDescManager getCubeDescManager() {
 return CubeDescManager.getInstance(getTestConfig());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json 
b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
new file mode 100644
index 000..c0e8568
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
@@ -0,0 +1,585 @@
+{
+  "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+  "last_modified" : 1479207711845,
+  "name" : "ut_large_dimension_number",
+  "model_name" : "ut_large_dimension_number",
+  "description" : "",
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "DEFAULT.WIDE_TABLE.A",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "A",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.B",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "B",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.C",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "C",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.D",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "D",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.E",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : 

[01/12] kylin git commit: KYLIN-2193 parameterise org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 a2366b0d6 -> 2107dc55f (forced update)


KYLIN-2193 parameterise 
org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e9a59d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e9a59d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e9a59d5

Branch: refs/heads/yang21-cdh5.7
Commit: 2e9a59d57712b6429e2bed71a14206e213162366
Parents: 61335d9
Author: Hongbin Ma 
Authored: Tue Nov 15 21:06:21 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 15 21:22:41 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  |  5 -
 .../storage/translate/DerivedFilterTranslator.java | 13 +
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e703a07..a91e42b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -480,7 +480,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 public float getCubeVisitTimeoutTimes() {
 return 
Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
 }
-
+
+public int getDerivedInThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20"));
+}
 public int getBadQueryStackTraceDepth() {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 60c3a01..54b4c31 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.translate;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.kv.RowKeyColumnOrder;
@@ -36,17 +37,18 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 /**
  * @author yangli9
- * 
  */
 public class DerivedFilterTranslator {
-
-private static final int IN_THRESHOLD = 5;
+
+private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
 
@@ -83,7 +85,10 @@ public class DerivedFilterTranslator {
 
 TupleFilter translated;
 boolean loosened;
-if (satisfyingHostRecords.size() > IN_THRESHOLD) {
+if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
+);
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);
 loosened = true;
 } else {

[10/12] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[12/12] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2107dc55
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2107dc55
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2107dc55

Branch: refs/heads/yang21-cdh5.7
Commit: 2107dc55f03d011f6d8c399a11aae0646fb79a27
Parents: a07a529
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Sat Nov 19 22:19:16 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2107dc55/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/2107dc55/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/2107dc55/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2107dc55/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[03/12] kylin git commit: fix compile

[mahongbin]

fix compile


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c0814f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c0814f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c0814f

Branch: refs/heads/yang21-cdh5.7
Commit: c0c0814f85509efe19c809b4f1fc22d2c63bf57a
Parents: b3e7eb7
Author: Hongbin Ma 
Authored: Wed Nov 16 10:17:28 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:28 2016 +0800

--
 .../apache/kylin/storage/translate/DerivedFilterTranslator.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c0814f/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 54b4c31..83fa32d 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -47,7 +47,7 @@ import com.google.common.collect.Sets;
  * @author yangli9
  */
 public class DerivedFilterTranslator {
-
+
 private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
@@ -86,7 +86,7 @@ public class DerivedFilterTranslator {
 TupleFilter translated;
 boolean loosened;
 if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
-logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}", //
 satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
 );
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);

[04/11] kylin git commit: KYLIN-2198 Add a framework to allow major changes in DimensionEncoding

[mahongbin]

KYLIN-2198 Add a framework to allow major changes in DimensionEncoding


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1acc419
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1acc419
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1acc419

Branch: refs/heads/yang21-hbase1.x
Commit: e1acc4192a982f897489f52d1bbc836a5c207da6
Parents: c0c0814
Author: Hongbin Ma 
Authored: Wed Nov 16 14:47:53 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:11:34 2016 +0800

--
 .../apache/kylin/common/util/JacksonBean.java   | 55 +++
 .../apache/kylin/common/util/JacksonTest.java   | 39 
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |  2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  | 13 ++-
 .../dimension/DimensionEncodingFactory.java | 97 +++-
 .../kylin/measure/topn/TopNMeasureType.java | 12 ++-
 .../kylin/rest/controller/CubeController.java   | 12 +--
 7 files changed, 198 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
new file mode 100644
index 000..42357f2
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
+public class JacksonBean {
+
+@JsonProperty("a")
+private String a;
+@JsonProperty("b")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+private int b;
+
+public String getA() {
+return a;
+}
+
+public void setA(String a) {
+this.a = a;
+}
+
+public int getB() {
+return b;
+}
+
+public void setB(int b) {
+this.b = b;
+}
+
+@Override
+public String toString() {
+return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
new file mode 100644
index 000..81be7eb
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+
+import org.junit.Test;
+

[02/11] kylin git commit: KYLIN-2191 fix integer encoding and refine UT

[mahongbin]

KYLIN-2191 fix integer encoding and refine UT

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3e7eb75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3e7eb75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3e7eb75

Branch: refs/heads/yang21-hbase1.x
Commit: b3e7eb751005b79c59958230832921ed4763e019
Parents: 2e9a59d
Author: Roger Shi 
Authored: Tue Nov 15 20:59:15 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:09 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |  85 +++-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  | 128 +++
 3 files changed, 129 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b3e7eb75/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index ffd43e5..87d37be 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -20,22 +20,68 @@ package org.apache.kylin.gridtable;
 
 import static org.junit.Assert.assertTrue;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Ordering;
 
 public class DimEncodingPreserveOrderTest {
+private static List successValue;
+private static List failValue;
+
+@BeforeClass
+public static void initTestValue() {
+successValue = new ArrayList<>();
+successValue.add(new long[] { -127, 0, 127 });
+successValue.add(new long[] { -32767, -127, 0, 127, 32767 });
+successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 
8388607 });
+successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 
127, 32767, 8388607, 2147483647L });
+successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, 
-32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L });
+successValue.add(new long[] { -140737488355327L, -549755813887L, 
-2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 
549755813887L, 140737488355327L });
+successValue.add(new long[] { -36028797018963967L, -140737488355327L, 
-549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 
2147483647L, 549755813887L, 140737488355327L, 36028797018963967L });
+successValue.add(new long[] { //
+-9223372036854775807L, //
+-36028797018963967L, //
+-140737488355327L, //
+-549755813887L, //
+-2147483647L, //
+-8388607, //
+-32767, //
+-127, //
+0, //
+127, // (2 ^ 7) - 1
+32767, // (2 ^ 15)  - 1
+8388607, // (2 ^ 23) - 1
+2147483647L, // (2 ^ 31) - 1
+549755813887L, // (2 ^ 39) - 1
+140737488355327L, // (2 ^ 47) - 1
+36028797018963967L, // (2 ^ 55) - 1
+9223372036854775807L }); // (2 ^ 63) - 1
+
+failValue = new ArrayList<>();
+failValue.add(new long[] { -128, 128 });
+failValue.add(new long[] { -32768, 32768 });
+failValue.add(new long[] { -8388608, 8388608 });
+failValue.add(new long[] { -2147483648L, 2147483648L });
+failValue.add(new long[] { -549755813888L, 549755813888L });
+failValue.add(new long[] { -140737488355328L, 140737488355328L });
+failValue.add(new long[] { -36028797018963968L, 36028797018963968L });
+failValue.add(new long[] { -9223372036854775808L });
+}
+
 @Test
 public void testOneMoreByteVLongDimEncPreserveOrder() {
+// TODO: better test
 OneMoreByteVLongDimEnc enc = new OneMoreByteVLongDimEnc(2);
 List encodedValues = Lists.newArrayList();
 

[09/11] kylin git commit: minor, fix UT

[mahongbin]

minor, fix UT

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61dd19f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61dd19f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61dd19f6

Branch: refs/heads/yang21-hbase1.x
Commit: 61dd19f6f64e2fdcb3b09de5f4b1c171d942ed1c
Parents: d87b7a5
Author: Roger Shi 
Authored: Thu Nov 17 17:08:17 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 17 17:54:45 2016 +0800

--
 .../common/util/LocalFileMetadataTestCase.java  |   1 +
 .../kylin/cube/AggregationGroupRuleTest.java|  37 +-
 .../org/apache/kylin/cube/CubeDescTest.java |  13 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |   7 +
 .../validation/rule/DictionaryRuleTest.java |   3 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 -
 ..._cube_desc_combination_int_overflow.json.bad | 398 +
 .../cube_desc/ut_large_dimension_number.json| 585 ---
 .../ut_large_dimension_number.json.bad  | 585 +++
 9 files changed, 1036 insertions(+), 991 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61dd19f6/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index c3e2538..d351041 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
 
 public static String LOCALMETA_TEST_DATA = 
"../examples/test_case_data/localmeta";
+public static String LOCALMETA_TEMP_DATA = "../examples/test_metadata/";
 
 @Override
 public void createTestMetadata() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/61dd19f6/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 7fea440..7903017 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -33,15 +33,31 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.validation.IValidatorRule;
 import org.apache.kylin.cube.model.validation.ValidateContext;
 import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
+import org.apache.kylin.metadata.MetadataManager;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
-public class AggregationGroupRuleTest {
+public class AggregationGroupRuleTest extends LocalFileMetadataTestCase{
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+MetadataManager.clearCache();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
 
 @Test
 public void testGoodDesc() throws IOException {
 AggregationGroupRule rule = getAggregationGroupRule();
 
-for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/").listFiles()) {
+for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + 
"/cube_desc/").listFiles()) {
+if (!f.getName().endsWith("json")) {
+continue;
+}
 CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), 
CubeDesc.class);
 ValidateContext vContext = new ValidateContext();
 rule.validate(desc, vContext);
@@ -59,7 +75,7 @@ public class AggregationGroupRuleTest {
 }
 };
 
-for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/").listFiles()) {
+for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + 
"/cube_desc/").listFiles()) {
 System.out.println(f.getName());
 CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), 
CubeDesc.class);
 ValidateContext vContext = new ValidateContext();
@@ -74,7 +90,7 @@ public class AggregationGroupRuleTest {
 public void testGoodDesc2() throws IOException {
 
 ValidateContext vContext = new 

[10/11] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[07/11] kylin git commit: KYLIN-2197 fix gg group buildMandatoryColumnMask

[mahongbin]

KYLIN-2197 fix gg group buildMandatoryColumnMask

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/02e723ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/02e723ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/02e723ae

Branch: refs/heads/yang21-hbase1.x
Commit: 02e723ae0c40847517c8e8c195cb723a5d70d275
Parents: ac4e9ec
Author: Li Yang 
Authored: Wed Nov 16 13:33:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  13 +
 .../cube_desc/ut_large_dimension_number.json| 585 +++
 .../model_desc/ut_large_dimension_number.json   |  26 +
 .../localmeta/table/DEFAULT.WIDE_TABLE.json | 217 +++
 5 files changed, 842 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 5f780d0..c7924c7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -124,7 +124,7 @@ public class AggregationGroup {
 for (String dim : mandatory_dims) {
 TblColRef hColumn = colNameAbbr.get(dim);
 Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
-mandatoryColumnMask |= 1 << index;
+mandatoryColumnMask |= (1L << index);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index ecb1200..bdceb0f 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -283,6 +284,18 @@ public class CuboidSchedulerTest extends 
LocalFileMetadataTestCase {
 assertEquals(cuboidScheduler.getCuboidCount(), sum);
 }
 
+@Test
+public void testCuboid_onlyBaseCuboid() {
+CubeDesc cube = 
getCubeDescManager().getCubeDesc("ut_large_dimension_number");
+CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
+assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+
+List spanningChild = 
scheduler.getSpanningCuboid(baseCuboid.getId());
+assertTrue(spanningChild.size() > 0);
+}
+
 public CubeDescManager getCubeDescManager() {
 return CubeDescManager.getInstance(getTestConfig());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json 
b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
new file mode 100644
index 000..c0e8568
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
@@ -0,0 +1,585 @@
+{
+  "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+  "last_modified" : 1479207711845,
+  "name" : "ut_large_dimension_number",
+  "model_name" : "ut_large_dimension_number",
+  "description" : "",
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "DEFAULT.WIDE_TABLE.A",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "A",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.B",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "B",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.C",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "C",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.D",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "D",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.E",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" 

[01/11] kylin git commit: KYLIN-2193 parameterise org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x e2adfdcfc -> a07a5292b (forced update)


KYLIN-2193 parameterise 
org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e9a59d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e9a59d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e9a59d5

Branch: refs/heads/yang21-hbase1.x
Commit: 2e9a59d57712b6429e2bed71a14206e213162366
Parents: 61335d9
Author: Hongbin Ma 
Authored: Tue Nov 15 21:06:21 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 15 21:22:41 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  |  5 -
 .../storage/translate/DerivedFilterTranslator.java | 13 +
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e703a07..a91e42b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -480,7 +480,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 public float getCubeVisitTimeoutTimes() {
 return 
Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
 }
-
+
+public int getDerivedInThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20"));
+}
 public int getBadQueryStackTraceDepth() {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 60c3a01..54b4c31 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.translate;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.kv.RowKeyColumnOrder;
@@ -36,17 +37,18 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 /**
  * @author yangli9
- * 
  */
 public class DerivedFilterTranslator {
-
-private static final int IN_THRESHOLD = 5;
+
+private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
 
@@ -83,7 +85,10 @@ public class DerivedFilterTranslator {
 
 TupleFilter translated;
 boolean loosened;
-if (satisfyingHostRecords.size() > IN_THRESHOLD) {
+if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
+);
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);
 loosened = true;
 } else {

[08/11] kylin git commit: KYLIN-2199 bug fix

[mahongbin]

KYLIN-2199 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d87b7a51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d87b7a51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d87b7a51

Branch: refs/heads/yang21-hbase1.x
Commit: d87b7a51282ffb2e678af65c20b5a395a2347d49
Parents: 9bd6a2a
Author: Hongbin Ma 
Authored: Thu Nov 17 16:46:59 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 16:48:53 2016 +0800

--
 .../org/apache/kylin/dimension/DimensionEncodingFactory.java   | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d87b7a51/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 242e003..b617a19 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -73,6 +73,12 @@ public abstract class DimensionEncodingFactory {
 
 Map result = Maps.newHashMap();
 for (Pair p : factoryMap.keySet()) {
+if (result.containsKey(p.getFirst())) {
+if (result.get(p.getFirst()) > p.getSecond()) {
+continue;//skip small versions
+}
+}
+
 result.put(p.getFirst(), p.getSecond());
 }
 result.put(DictionaryDimEnc.ENCODING_NAME, 1);

[11/11] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a07a5292
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a07a5292
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a07a5292

Branch: refs/heads/yang21-hbase1.x
Commit: a07a5292b6352940cf0e5993890bc0b057ce3c3d
Parents: 61dd19f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Nov 19 22:17:14 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-   

[03/11] kylin git commit: fix compile

[mahongbin]

fix compile


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c0814f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c0814f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c0814f

Branch: refs/heads/yang21-hbase1.x
Commit: c0c0814f85509efe19c809b4f1fc22d2c63bf57a
Parents: b3e7eb7
Author: Hongbin Ma 
Authored: Wed Nov 16 10:17:28 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:28 2016 +0800

--
 .../apache/kylin/storage/translate/DerivedFilterTranslator.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c0814f/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 54b4c31..83fa32d 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -47,7 +47,7 @@ import com.google.common.collect.Sets;
  * @author yangli9
  */
 public class DerivedFilterTranslator {
-
+
 private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
@@ -86,7 +86,7 @@ public class DerivedFilterTranslator {
 TupleFilter translated;
 boolean loosened;
 if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
-logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}", //
 satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
 );
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);

[06/11] kylin git commit: KYLIN-2201 fix combination check overflow error

[mahongbin]

KYLIN-2201 fix combination check overflow error

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9bd6a2ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9bd6a2ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9bd6a2ab

Branch: refs/heads/yang21-hbase1.x
Commit: 9bd6a2ab6b5fd8f11a80198012a519bfe4edac08
Parents: 02e723a
Author: Roger Shi 
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   |   6 +-
 .../kylin/cube/AggregationGroupRuleTest.java|  11 +
 .../org/apache/kylin/cube/CubeDescTest.java |   7 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 +++
 4 files changed, 419 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..c914ebb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -580,7 +580,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 throw new IllegalStateException("Aggregation group " + index + 
" select rule field not set");
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(includeDims, agg.getIncludes());
 
@@ -598,7 +598,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set jointDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
 if (jointDimsList.size() > 0) {
-combination = combination * (1 << jointDimsList.size());
+combination = combination * (1L << jointDimsList.size());
 }
 
 if (!includeDims.containsAll(mandatoryDims) || 
!includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) 
{
@@ -612,7 +612,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (combination > config.getCubeAggrGroupMaxCombination()) {
 String msg = "Aggregation group " + index + " has too many 
combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 
'kylin.cube.aggrgroup.max.combination' to a bigger value.";

http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 6c3d544..7fea440 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -115,6 +116,16 @@ public class AggregationGroupRuleTest {
 assertEquals("Aggregation group 0 joint columns overlap with more than 
1 dim in same hierarchy", (vContext.getResults()[0].getMessage()));
 }
 
+@Test
+public void testCombinationIntOverflow() throws IOException {
+ValidateContext vContext = new ValidateContext();
+CubeDesc desc = JsonUtil.readValue(new 
FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
+
+IValidatorRule rule = getAggregationGroupRule();
+rule.validate(desc, vContext);
+assertEquals(1, vContext.getResults().length);
+}
+
 public AggregationGroupRule getAggregationGroupRule() {
 AggregationGroupRule rule = new AggregationGroupRule() {
 @Override

[1/3] kylin git commit: KYLIN-2201 fix combination check overflow error

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21 ac4e9ec56 -> d87b7a512


KYLIN-2201 fix combination check overflow error

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9bd6a2ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9bd6a2ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9bd6a2ab

Branch: refs/heads/yang21
Commit: 9bd6a2ab6b5fd8f11a80198012a519bfe4edac08
Parents: 02e723a
Author: Roger Shi 
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   |   6 +-
 .../kylin/cube/AggregationGroupRuleTest.java|  11 +
 .../org/apache/kylin/cube/CubeDescTest.java |   7 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 +++
 4 files changed, 419 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..c914ebb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -580,7 +580,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 throw new IllegalStateException("Aggregation group " + index + 
" select rule field not set");
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(includeDims, agg.getIncludes());
 
@@ -598,7 +598,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set jointDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
 if (jointDimsList.size() > 0) {
-combination = combination * (1 << jointDimsList.size());
+combination = combination * (1L << jointDimsList.size());
 }
 
 if (!includeDims.containsAll(mandatoryDims) || 
!includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) 
{
@@ -612,7 +612,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (combination > config.getCubeAggrGroupMaxCombination()) {
 String msg = "Aggregation group " + index + " has too many 
combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 
'kylin.cube.aggrgroup.max.combination' to a bigger value.";

http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 6c3d544..7fea440 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -115,6 +116,16 @@ public class AggregationGroupRuleTest {
 assertEquals("Aggregation group 0 joint columns overlap with more than 
1 dim in same hierarchy", (vContext.getResults()[0].getMessage()));
 }
 
+@Test
+public void testCombinationIntOverflow() throws IOException {
+ValidateContext vContext = new ValidateContext();
+CubeDesc desc = JsonUtil.readValue(new 
FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
+
+IValidatorRule rule = getAggregationGroupRule();
+rule.validate(desc, vContext);
+assertEquals(1, vContext.getResults().length);
+}
+
 public AggregationGroupRule getAggregationGroupRule() {
 AggregationGroupRule rule = new AggregationGroupRule() {
 @Override

[2/3] kylin git commit: KYLIN-2197 fix gg group buildMandatoryColumnMask

[mahongbin]

KYLIN-2197 fix gg group buildMandatoryColumnMask

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/02e723ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/02e723ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/02e723ae

Branch: refs/heads/yang21
Commit: 02e723ae0c40847517c8e8c195cb723a5d70d275
Parents: ac4e9ec
Author: Li Yang 
Authored: Wed Nov 16 13:33:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  13 +
 .../cube_desc/ut_large_dimension_number.json| 585 +++
 .../model_desc/ut_large_dimension_number.json   |  26 +
 .../localmeta/table/DEFAULT.WIDE_TABLE.json | 217 +++
 5 files changed, 842 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 5f780d0..c7924c7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -124,7 +124,7 @@ public class AggregationGroup {
 for (String dim : mandatory_dims) {
 TblColRef hColumn = colNameAbbr.get(dim);
 Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
-mandatoryColumnMask |= 1 << index;
+mandatoryColumnMask |= (1L << index);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index ecb1200..bdceb0f 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -283,6 +284,18 @@ public class CuboidSchedulerTest extends 
LocalFileMetadataTestCase {
 assertEquals(cuboidScheduler.getCuboidCount(), sum);
 }
 
+@Test
+public void testCuboid_onlyBaseCuboid() {
+CubeDesc cube = 
getCubeDescManager().getCubeDesc("ut_large_dimension_number");
+CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
+assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+
+List spanningChild = 
scheduler.getSpanningCuboid(baseCuboid.getId());
+assertTrue(spanningChild.size() > 0);
+}
+
 public CubeDescManager getCubeDescManager() {
 return CubeDescManager.getInstance(getTestConfig());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json 
b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
new file mode 100644
index 000..c0e8568
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
@@ -0,0 +1,585 @@
+{
+  "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+  "last_modified" : 1479207711845,
+  "name" : "ut_large_dimension_number",
+  "model_name" : "ut_large_dimension_number",
+  "description" : "",
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "DEFAULT.WIDE_TABLE.A",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "A",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.B",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "B",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.C",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "C",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.D",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "D",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.E",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "E",
+  

[3/3] kylin git commit: KYLIN-2199 bug fix

[mahongbin]

KYLIN-2199 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d87b7a51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d87b7a51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d87b7a51

Branch: refs/heads/yang21
Commit: d87b7a51282ffb2e678af65c20b5a395a2347d49
Parents: 9bd6a2a
Author: Hongbin Ma 
Authored: Thu Nov 17 16:46:59 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 16:48:53 2016 +0800

--
 .../org/apache/kylin/dimension/DimensionEncodingFactory.java   | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d87b7a51/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 242e003..b617a19 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -73,6 +73,12 @@ public abstract class DimensionEncodingFactory {
 
 Map result = Maps.newHashMap();
 for (Pair p : factoryMap.keySet()) {
+if (result.containsKey(p.getFirst())) {
+if (result.get(p.getFirst()) > p.getSecond()) {
+continue;//skip small versions
+}
+}
+
 result.put(p.getFirst(), p.getSecond());
 }
 result.put(DictionaryDimEnc.ENCODING_NAME, 1);

kylin git commit: KYLIN-2199 bug fix

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 28276e940 -> 9cf480bc6


KYLIN-2199 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9cf480bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9cf480bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9cf480bc

Branch: refs/heads/master
Commit: 9cf480bc64b3790765a75f5312187df318aadc40
Parents: 28276e9
Author: Hongbin Ma 
Authored: Thu Nov 17 16:46:59 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 16:47:03 2016 +0800

--
 .../org/apache/kylin/dimension/DimensionEncodingFactory.java   | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9cf480bc/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 37d1014..843f4cd 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -73,6 +73,12 @@ public abstract class DimensionEncodingFactory {
 
 Map result = Maps.newHashMap();
 for (Pair p : factoryMap.keySet()) {
+if (result.containsKey(p.getFirst())) {
+if (result.get(p.getFirst()) > p.getSecond()) {
+continue;//skip small versions
+}
+}
+
 result.put(p.getFirst(), p.getSecond());
 }
 result.put(DictionaryDimEnc.ENCODING_NAME, 1);

[1/2] kylin git commit: KYLIN-2201 fix combination check overflow error

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 3b1850eae -> 00ba74327


KYLIN-2201 fix combination check overflow error

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00ba7432
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00ba7432
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00ba7432

Branch: refs/heads/master
Commit: 00ba74327a0322e1ce62d3c1cf7953ab21ae9407
Parents: f3d19f6
Author: Roger Shi 
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:28:06 2016 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   |   6 +-
 .../validation/rule/AggregationGroupRule.java   |   4 +-
 .../kylin/cube/AggregationGroupRuleTest.java|  11 +
 .../org/apache/kylin/cube/CubeDescTest.java |   7 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 +++
 5 files changed, 421 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/00ba7432/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 7dad87b..ce6389f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -579,7 +579,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 throw new IllegalStateException("Aggregation group " + index + 
" select rule field not set");
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(includeDims, agg.getIncludes());
 
@@ -597,7 +597,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set jointDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
 if (jointDimsList.size() > 0) {
-combination = combination * (1 << jointDimsList.size());
+combination = combination * (1L << jointDimsList.size());
 }
 
 if (!includeDims.containsAll(mandatoryDims) || 
!includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) 
{
@@ -619,7 +619,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (combination > config.getCubeAggrGroupMaxCombination()) {
 String msg = "Aggregation group " + index + " has too many 
combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 
'kylin.cube.aggrgroup.max.combination' to a bigger value.";

http://git-wip-us.apache.org/repos/asf/kylin/blob/00ba7432/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index c1a062a..a5cf407 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -62,7 +62,7 @@ public class AggregationGroupRule implements 
IValidatorRule {
 continue;
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 if (agg.getIncludes() != null) {
 for (String include : agg.getIncludes()) {
@@ -115,7 +115,7 @@ public class AggregationGroupRule implements 
IValidatorRule {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
 Set intersection = new HashSet<>(mandatoryDims);

[2/2] kylin git commit: KYLIN-2197 fix gg group buildMandatoryColumnMask

[mahongbin]

KYLIN-2197 fix gg group buildMandatoryColumnMask

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f3d19f6d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f3d19f6d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f3d19f6d

Branch: refs/heads/master
Commit: f3d19f6da20fa415eac641b8daf6081b3eaaf9d4
Parents: 3b1850e
Author: Li Yang 
Authored: Wed Nov 16 13:33:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:28:06 2016 +0800

--
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  13 +
 .../cube_desc/ut_large_dimension_number.json| 585 +++
 .../model_desc/ut_large_dimension_number.json   |  26 +
 .../localmeta/table/DEFAULT.WIDE_TABLE.json | 217 +++
 5 files changed, 842 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f3d19f6d/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 027d185..ab63c00 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -140,7 +140,7 @@ public class AggregationGroup {
 for (String dim : mandatory_dims) {
 TblColRef hColumn = cubeDesc.getModel().findColumn(dim);
 Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
-mandatoryColumnMask |= 1 << index;
+mandatoryColumnMask |= (1L << index);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f3d19f6d/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index ecb1200..bdceb0f 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -283,6 +284,18 @@ public class CuboidSchedulerTest extends 
LocalFileMetadataTestCase {
 assertEquals(cuboidScheduler.getCuboidCount(), sum);
 }
 
+@Test
+public void testCuboid_onlyBaseCuboid() {
+CubeDesc cube = 
getCubeDescManager().getCubeDesc("ut_large_dimension_number");
+CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
+assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+
+List spanningChild = 
scheduler.getSpanningCuboid(baseCuboid.getId());
+assertTrue(spanningChild.size() > 0);
+}
+
 public CubeDescManager getCubeDescManager() {
 return CubeDescManager.getInstance(getTestConfig());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f3d19f6d/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json 
b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
new file mode 100644
index 000..c0e8568
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
@@ -0,0 +1,585 @@
+{
+  "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+  "last_modified" : 1479207711845,
+  "name" : "ut_large_dimension_number",
+  "model_name" : "ut_large_dimension_number",
+  "description" : "",
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "DEFAULT.WIDE_TABLE.A",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "A",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.B",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "B",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.C",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "C",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.D",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "D",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.E",
+"table" : "DEFAULT.WIDE_TABLE",
+

[2/4] kylin git commit: fix compile

[mahongbin]

fix compile


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c0814f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c0814f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c0814f

Branch: refs/heads/yang21
Commit: c0c0814f85509efe19c809b4f1fc22d2c63bf57a
Parents: b3e7eb7
Author: Hongbin Ma 
Authored: Wed Nov 16 10:17:28 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:28 2016 +0800

--
 .../apache/kylin/storage/translate/DerivedFilterTranslator.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c0814f/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 54b4c31..83fa32d 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -47,7 +47,7 @@ import com.google.common.collect.Sets;
  * @author yangli9
  */
 public class DerivedFilterTranslator {
-
+
 private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
@@ -86,7 +86,7 @@ public class DerivedFilterTranslator {
 TupleFilter translated;
 boolean loosened;
 if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
-logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}", //
 satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
 );
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);

[4/4] kylin git commit: KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198

[mahongbin]

KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac4e9ec5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac4e9ec5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac4e9ec5

Branch: refs/heads/yang21
Commit: ac4e9ec563b6e14e387c5803d441c8dbcbc06f87
Parents: e1acc41
Author: Hongbin Ma 
Authored: Wed Nov 16 14:48:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:14:24 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |   4 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   9 +-
 .../apache/kylin/dimension/IntegerDimEncV2.java | 228 +++
 .../apache/kylin/dimension/IntDimEncTest.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  |  18 +-
 5 files changed, 248 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 87d37be..d572e56 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.IntegerDimEncV2;
 import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -99,7 +99,7 @@ public class DimEncodingPreserveOrderTest {
 @Test
 public void testVLongDimEncPreserveOrder() {
 for (int i = 1; i <= successValue.size(); i++) {
-IntegerDimEnc enc = new IntegerDimEnc(i);
+IntegerDimEncV2 enc = new IntegerDimEncV2(i);
 List encodedValues = Lists.newArrayList();
 for (long value : successValue.get(i - 1)) {
 encodedValues.add(encode(enc, value));

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index e55a0a8..983af9a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -31,8 +31,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * replacement for IntegerDimEnc, the diff is VLongDimEnc supports negative 
values
+ * replacement for IntDimEnc, the diff is IntegerDimEnc supports negative 
values
+ * for IntegerDimEnc(N), the supported range is (-2^(8*N-1),2^(8*N-1))
+ *
+ * -2^(8*N-1) is not supported because the slot is reserved for null values.
+ * -2^(8*N-1) will be encoded with warn, and its output will be null
  */
+@Deprecated//due to a fatal bug (KYLIN-2191)
 public class IntegerDimEnc extends DimensionEncoding {
 private static final long serialVersionUID = 1L;
 
@@ -127,7 +132,7 @@ public class IntegerDimEnc extends DimensionEncoding {
 
 //only take useful bytes
 integer = integer & MASK[fixedLen];
-boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
+boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0;
 if (!positive) {
 integer |= (~MASK[fixedLen]);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
new file mode 100644
index 000..1a54664
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the 

[2/2] kylin git commit: KYLIN-2198 Add a framework to allow major changes in DimensionEncoding

[mahongbin]

KYLIN-2198 Add a framework to allow major changes in DimensionEncoding


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/caf7040f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/caf7040f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/caf7040f

Branch: refs/heads/master
Commit: caf7040fc745ccca2525300979fce7785d6f0969
Parents: 21a1c5c
Author: Hongbin Ma 
Authored: Wed Nov 16 14:47:53 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 10:56:36 2016 +0800

--
 .../apache/kylin/common/util/JacksonBean.java   |  55 ++
 .../apache/kylin/common/util/JacksonTest.java   |  39 +++
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |   2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  13 ++-
 .../dimension/DimensionEncodingFactory.java | 102 ++-
 .../kylin/measure/topn/TopNMeasureType.java |  12 ++-
 .../kylin/rest/controller/CubeController.java   |  13 ++-
 7 files changed, 202 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
new file mode 100644
index 000..42357f2
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
+public class JacksonBean {
+
+@JsonProperty("a")
+private String a;
+@JsonProperty("b")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+private int b;
+
+public String getA() {
+return a;
+}
+
+public void setA(String a) {
+this.a = a;
+}
+
+public int getB() {
+return b;
+}
+
+public void setB(int b) {
+this.b = b;
+}
+
+@Override
+public String toString() {
+return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
new file mode 100644
index 000..81be7eb
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+
+import org.junit.Test;
+

[1/2] kylin git commit: KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 21a1c5c09 -> 4fcaa8ac3


KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4fcaa8ac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4fcaa8ac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4fcaa8ac

Branch: refs/heads/master
Commit: 4fcaa8ac3fdeb7709ba53b531393ab88a71ecdd5
Parents: caf7040
Author: Hongbin Ma 
Authored: Wed Nov 16 14:48:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 10:56:36 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |   4 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   8 +-
 .../apache/kylin/dimension/IntegerDimEncV2.java | 228 +++
 .../apache/kylin/dimension/IntDimEncTest.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  |  18 +-
 5 files changed, 247 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4fcaa8ac/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 87d37be..d572e56 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.IntegerDimEncV2;
 import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -99,7 +99,7 @@ public class DimEncodingPreserveOrderTest {
 @Test
 public void testVLongDimEncPreserveOrder() {
 for (int i = 1; i <= successValue.size(); i++) {
-IntegerDimEnc enc = new IntegerDimEnc(i);
+IntegerDimEncV2 enc = new IntegerDimEncV2(i);
 List encodedValues = Lists.newArrayList();
 for (long value : successValue.get(i - 1)) {
 encodedValues.add(encode(enc, value));

http://git-wip-us.apache.org/repos/asf/kylin/blob/4fcaa8ac/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 460ca65..c417e37 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -32,8 +32,12 @@ import org.slf4j.LoggerFactory;
 
 /**
  * replacement for IntDimEnc, the diff is IntegerDimEnc supports negative 
values
- * for IntegerDimEnc(N), the supported range is [-2^(8N-1),2^(8N-1)]
+ * for IntegerDimEnc(N), the supported range is (-2^(8*N-1),2^(8*N-1))
+ *
+ * -2^(8*N-1) is not supported because the slot is reserved for null values.
+ * -2^(8*N-1) will be encoded with warn, and its output will be null
  */
+@Deprecated//due to a fatal bug (KYLIN-2191) and the limitation of not able to 
represent -2^(8N-1)
 public class IntegerDimEnc extends DimensionEncoding {
 private static final long serialVersionUID = 1L;
 
@@ -128,7 +132,7 @@ public class IntegerDimEnc extends DimensionEncoding {
 
 //only take useful bytes
 integer = integer & MASK[fixedLen];
-boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
+boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0;
 if (!positive) {
 integer |= (~MASK[fixedLen]);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4fcaa8ac/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
new file mode 100644
index 000..1a54664
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional 

kylin git commit: fix compile

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master acacb67db -> dd9c2c9f5


fix compile


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dd9c2c9f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dd9c2c9f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dd9c2c9f

Branch: refs/heads/master
Commit: dd9c2c9f504e462f71ed93cf317ef4a46bf6237f
Parents: acacb67
Author: Hongbin Ma 
Authored: Wed Nov 16 10:17:28 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 16 10:17:39 2016 +0800

--
 .../apache/kylin/storage/translate/DerivedFilterTranslator.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dd9c2c9f/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 54b4c31..83fa32d 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -47,7 +47,7 @@ import com.google.common.collect.Sets;
  * @author yangli9
  */
 public class DerivedFilterTranslator {
-
+
 private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
@@ -86,7 +86,7 @@ public class DerivedFilterTranslator {
 TupleFilter translated;
 boolean loosened;
 if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
-logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}", //
 satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
 );
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);

kylin git commit: KYLIN-2191 fix integer encoding and refine UT

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master e68b25796 -> ab6e08351


KYLIN-2191 fix integer encoding and refine UT

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab6e0835
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab6e0835
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab6e0835

Branch: refs/heads/master
Commit: ab6e083517a16bece87db34d9f7f22a5df0662ee
Parents: e68b257
Author: Roger Shi 
Authored: Tue Nov 15 20:59:15 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 15 21:33:45 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |  85 +++-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  | 128 +++
 3 files changed, 129 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ab6e0835/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index ffd43e5..87d37be 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -20,22 +20,68 @@ package org.apache.kylin.gridtable;
 
 import static org.junit.Assert.assertTrue;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Ordering;
 
 public class DimEncodingPreserveOrderTest {
+private static List successValue;
+private static List failValue;
+
+@BeforeClass
+public static void initTestValue() {
+successValue = new ArrayList<>();
+successValue.add(new long[] { -127, 0, 127 });
+successValue.add(new long[] { -32767, -127, 0, 127, 32767 });
+successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 
8388607 });
+successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 
127, 32767, 8388607, 2147483647L });
+successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, 
-32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L });
+successValue.add(new long[] { -140737488355327L, -549755813887L, 
-2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 
549755813887L, 140737488355327L });
+successValue.add(new long[] { -36028797018963967L, -140737488355327L, 
-549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 
2147483647L, 549755813887L, 140737488355327L, 36028797018963967L });
+successValue.add(new long[] { //
+-9223372036854775807L, //
+-36028797018963967L, //
+-140737488355327L, //
+-549755813887L, //
+-2147483647L, //
+-8388607, //
+-32767, //
+-127, //
+0, //
+127, // (2 ^ 7) - 1
+32767, // (2 ^ 15)  - 1
+8388607, // (2 ^ 23) - 1
+2147483647L, // (2 ^ 31) - 1
+549755813887L, // (2 ^ 39) - 1
+140737488355327L, // (2 ^ 47) - 1
+36028797018963967L, // (2 ^ 55) - 1
+9223372036854775807L }); // (2 ^ 63) - 1
+
+failValue = new ArrayList<>();
+failValue.add(new long[] { -128, 128 });
+failValue.add(new long[] { -32768, 32768 });
+failValue.add(new long[] { -8388608, 8388608 });
+failValue.add(new long[] { -2147483648L, 2147483648L });
+failValue.add(new long[] { -549755813888L, 549755813888L });
+failValue.add(new long[] { -140737488355328L, 140737488355328L });
+failValue.add(new long[] { -36028797018963968L, 36028797018963968L });
+failValue.add(new long[] { -9223372036854775808L });
+}
+
 @Test
 public void testOneMoreByteVLongDimEncPreserveOrder() {
+// TODO: better test
 OneMoreByteVLongDimEnc enc = new OneMoreByteVLongDimEnc(2);
 

kylin git commit: KYLIN-2193 parameterise org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21 61335d9ce -> 2e9a59d57


KYLIN-2193 parameterise 
org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e9a59d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e9a59d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e9a59d5

Branch: refs/heads/yang21
Commit: 2e9a59d57712b6429e2bed71a14206e213162366
Parents: 61335d9
Author: Hongbin Ma 
Authored: Tue Nov 15 21:06:21 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 15 21:22:41 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  |  5 -
 .../storage/translate/DerivedFilterTranslator.java | 13 +
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e703a07..a91e42b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -480,7 +480,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 public float getCubeVisitTimeoutTimes() {
 return 
Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
 }
-
+
+public int getDerivedInThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20"));
+}
 public int getBadQueryStackTraceDepth() {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 60c3a01..54b4c31 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.translate;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.kv.RowKeyColumnOrder;
@@ -36,17 +37,18 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 /**
  * @author yangli9
- * 
  */
 public class DerivedFilterTranslator {
-
-private static final int IN_THRESHOLD = 5;
+
+private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
 
@@ -83,7 +85,10 @@ public class DerivedFilterTranslator {
 
 TupleFilter translated;
 boolean loosened;
-if (satisfyingHostRecords.size() > IN_THRESHOLD) {
+if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
+);
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);
 loosened = true;
 } else {

[kylin] Git Push Summary

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/tmp [deleted] cba24cafd

kylin git commit: KYLIN-2193 parameterise org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 2c2970feb -> e68b25796


KYLIN-2193 parameterise 
org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e68b2579
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e68b2579
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e68b2579

Branch: refs/heads/master
Commit: e68b257968a03809e114f6871394c4b55b05562f
Parents: 2c2970f
Author: Hongbin Ma 
Authored: Tue Nov 15 21:06:21 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 15 21:06:25 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  |  5 -
 .../storage/translate/DerivedFilterTranslator.java | 13 +
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e68b2579/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ee9f57c..d44ff96 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -486,6 +486,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan.threshold", 
"1000"));
 }
 
+public int getDerivedInThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20"));
+}
+
 public int getBadQueryStackTraceDepth() {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }
@@ -634,7 +638,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
this.getOptional("kylin.job.hive.database.for.intermediatetable", "default");
 }
 
-
 public boolean isHiveRedistributeEnabled() {
 return 
Boolean.parseBoolean(this.getOptional("kylin.job.hive.intermediatetable.redistribute.enabled",
 "true"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e68b2579/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 60c3a01..54b4c31 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.translate;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.kv.RowKeyColumnOrder;
@@ -36,17 +37,18 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 /**
  * @author yangli9
- * 
  */
 public class DerivedFilterTranslator {
-
-private static final int IN_THRESHOLD = 5;
+
+private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
 
@@ -83,7 +85,10 @@ public class DerivedFilterTranslator {
 
 TupleFilter translated;
 boolean loosened;
-if (satisfyingHostRecords.size() > IN_THRESHOLD) {
+if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
+);
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);
 loosened = true;
 } else {

kylin git commit: update comment for IntegerDimEnc

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 13e1e75f8 -> 2a101f663


update comment for IntegerDimEnc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2a101f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2a101f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2a101f66

Branch: refs/heads/master
Commit: 2a101f663ef3c25942a53a35d9e070afdbf97732
Parents: 13e1e75
Author: Hongbin Ma 
Authored: Mon Nov 14 22:46:25 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 22:46:25 2016 +0800

--
 .../src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2a101f66/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index a8cb510..dc9dff7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -31,7 +31,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * replacement for IntegerDimEnc, the diff is VLongDimEnc supports negative 
values
+ * replacement for IntDimEnc, the diff is IntegerDimEnc supports negative 
values
+ * for IntegerDimEnc(N), the supported range is [-2^(8N-1),2^(8N-1)]
  */
 public class IntegerDimEnc extends DimensionEncoding {
 private static final long serialVersionUID = 1L;

[1/2] kylin git commit: KYLIN-2181 fail cube which uses fixedlength for integer/number

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 07d892124 -> 8822e78af


KYLIN-2181 fail cube which uses fixedlength for integer/number


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8822e78a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8822e78a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8822e78a

Branch: refs/heads/master
Commit: 8822e78af041dbcd00f89c5e3ef65a712d394d76
Parents: 8ed3fbc
Author: Hongbin Ma 
Authored: Mon Nov 14 18:27:21 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 18:27:25 2016 +0800

--
 .../main/java/org/apache/kylin/cube/model/RowKeyColDesc.java| 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8822e78a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 9e2cb48..6192d76 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -23,6 +23,7 @@ import org.apache.kylin.dimension.DateDimEnc;
 import org.apache.kylin.dimension.DictionaryDimEnc;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
+import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.dimension.TimeDimEnc;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -61,7 +62,7 @@ public class RowKeyColDesc {
 column = column.toUpperCase();
 bitIndex = index;
 colRef = cubeDesc.getModel().findColumn(column);
-Preconditions.checkArgument(colRef != null, "Cannot find rowkey column 
%s in cube %s", column,  cubeDesc);
+Preconditions.checkArgument(colRef != null, "Cannot find rowkey column 
%s in cube %s", column, cubeDesc);
 
 Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
 Object[] encodingConf = 
DimensionEncoding.parseEncodingConf(this.encoding);
@@ -86,6 +87,8 @@ public class RowKeyColDesc {
 throw new IllegalArgumentException(colRef + " type is " + type + " 
and cannot apply date encoding");
 if (TimeDimEnc.ENCODING_NAME.equals(encodingName) && 
type.isTimeFamily() == false)
 throw new IllegalArgumentException(colRef + " type is " + type + " 
and cannot apply time encoding");
+if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && 
(type.isIntegerFamily() || type.isNumberFamily()))
+throw new IllegalArgumentException(colRef + " type is " + type + " 
and cannot apply fixed_length encoding");
 }
 
 public String getEncoding() {

[2/2] kylin git commit: add comments to KylinVersion

[mahongbin]

add comments to KylinVersion


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ed3fbc3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ed3fbc3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ed3fbc3

Branch: refs/heads/master
Commit: 8ed3fbc353965da9df7f18f53e84f0515200708e
Parents: 07d8921
Author: Hongbin Ma 
Authored: Mon Nov 14 17:40:33 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 18:27:25 2016 +0800

--
 .../main/java/org/apache/kylin/common/KylinVersion.java  | 11 +++
 1 file changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8ed3fbc3/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 0e4147a..b7c3804 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -73,6 +73,17 @@ public class KylinVersion {
 
 private static final Set SIGNATURE_INCOMPATIBLE_REVISIONS = 
new HashSet();
 
+/**
+ * 1.5.1 is actually compatible with 1.5.0's cube. However the "calculate 
signature" method in 1.5.1 code base somehow 
+ * gives different signature values for 1.5.0 cubes. To prevent from users 
having to take care of this mess, as people 
+ * usually won't expect to take lots of efforts for small upgrade (from 
1.5.0 to 1.5.1), a special list of 
+ * SIGNATURE_INCOMPATIBLE_REVISIONS is introduced to silently take care of 
such legacy cubes.
+ * 
+ * We should NEVER add new stuff to SIGNATURE_INCOMPATIBLE_REVISIONS. 
"calculate signature" should always return consistent values
+ * to compatible versions. If it's impossible to maintain consistent 
signatures between upgrade, we should increase the minor version,
+ * e.g. it's better to skip 1.5.1 and use 1.6.0 as the next release 
version to 1.5.0, or even to use 2.0.0, as people tends to accept 
+ * doing more (e.g. Having to use sth like a metastore upgrade tool when 
upgrading Kylin)
+ */
 static {
 SIGNATURE_INCOMPATIBLE_REVISIONS.add(new KylinVersion("1.5.1"));
 }

kylin git commit: KYLIN-2181 remove integer as fixed_length in test_kylin_cube_with_slr_empty desc

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master b6d8e8849 -> 38dfbb3ef


KYLIN-2181 remove integer as fixed_length in test_kylin_cube_with_slr_empty desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38dfbb3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38dfbb3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38dfbb3e

Branch: refs/heads/master
Commit: 38dfbb3ef31ca0490647aacf0666dce0f119835b
Parents: b6d8e88
Author: Hongbin Ma 
Authored: Mon Nov 14 13:54:12 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 13:54:12 2016 +0800

--
 .../localmeta/cube_desc/test_kylin_cube_with_slr_desc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/38dfbb3e/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index 4064fcb..9c9924c 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -120,7 +120,7 @@
   "encoding" : "dict"
 }, {
   "column" : "leaf_categ_id",
-  "encoding" : "fixed_length:18"
+  "encoding" : "dict"
 }, {
   "column" : "meta_categ_name",
   "encoding" : "dict"

[1/4] kylin git commit: KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers in the rowkey [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 5f867e218 -> 9a4a3bf08 (forced update)


KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding 
for integers in the rowkey


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61335d9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61335d9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61335d9c

Branch: refs/heads/yang21-cdh5.7
Commit: 61335d9ce8ff193bd22e881855d0c68e48896ca9
Parents: c23205d
Author: Hongbin Ma 
Authored: Mon Nov 14 09:12:51 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:12:51 2016 +0800

--
 .../gtrecord/GTCubeStorageQueryBase.java| 22 ++--
 1 file changed, 20 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61335d9c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index b51af59..d648469 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -34,7 +34,9 @@ import org.apache.kylin.cube.RawQueryLastHacker;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -120,7 +122,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 //set whether to aggr at storage
 context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, 
groupsD, singleValuesD));
 // set limit push down
-enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, 
groupsD, filter, sqlDigest.aggregations, context);
+enableStorageLimitIfPossible(cubeInstance.getDescriptor(), cuboid, 
groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, 
context);
 context.setFinalPushDownLimit(cubeInstance);
 // set cautious threshold to prevent out of memory
 setThresholdIfNecessary(dimensionsD, metrics, context);
@@ -412,9 +414,25 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 }
 }
 
-private void enableStorageLimitIfPossible(Cuboid cuboid, 
Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
+private void enableStorageLimitIfPossible(CubeDesc cubeDesc, Cuboid 
cuboid, Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
 boolean possible = true;
 
+for (TblColRef col : cuboid.getColumns()) {
+RowKeyColDesc rowKeyDesc = cubeDesc.getRowkey().getColDesc(col);
+if (rowKeyDesc != null) {
+String encodingName = rowKeyDesc.getEncodingName();
+
+//for numbers, if it's fixed_length encoding, order is no 
longer preserved
+if (encodingName != null && 
encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && //
+(col.getType().isNumberFamily() || 
col.getType().isIntegerFamily())) {
+possible = false;
+logger.info("Storage limit push down is impossible because 
integer/number is encoded as fixedlength");
+}
+} else {
+logger.warn("RowKeyColDesc for {} does not exist", col);
+}
+}
+
 boolean goodFilter = filter == null || 
TupleFilter.isEvaluableRecursively(filter);
 if (!goodFilter) {
 possible = false;

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a4a3bf0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a4a3bf0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a4a3bf0

Branch: refs/heads/yang21-cdh5.7
Commit: 9a4a3bf083dbe1f2199c913ea0749dc914544c65
Parents: b28fe63
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:18:18 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a4a3bf0/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a4a3bf0/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a4a3bf0/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a4a3bf0/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b28fe630
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b28fe630
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b28fe630

Branch: refs/heads/yang21-cdh5.7
Commit: b28fe630bea9d695f0c970ed8024c5327fa95160
Parents: 61335d9
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:16:37 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
- 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/3] kylin git commit: KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers in the rowkey [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 130c26df9 -> b28fe630b (forced update)


KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding 
for integers in the rowkey


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61335d9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61335d9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61335d9c

Branch: refs/heads/yang21-hbase1.x
Commit: 61335d9ce8ff193bd22e881855d0c68e48896ca9
Parents: c23205d
Author: Hongbin Ma 
Authored: Mon Nov 14 09:12:51 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:12:51 2016 +0800

--
 .../gtrecord/GTCubeStorageQueryBase.java| 22 ++--
 1 file changed, 20 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61335d9c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index b51af59..d648469 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -34,7 +34,9 @@ import org.apache.kylin.cube.RawQueryLastHacker;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -120,7 +122,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 //set whether to aggr at storage
 context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, 
groupsD, singleValuesD));
 // set limit push down
-enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, 
groupsD, filter, sqlDigest.aggregations, context);
+enableStorageLimitIfPossible(cubeInstance.getDescriptor(), cuboid, 
groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, 
context);
 context.setFinalPushDownLimit(cubeInstance);
 // set cautious threshold to prevent out of memory
 setThresholdIfNecessary(dimensionsD, metrics, context);
@@ -412,9 +414,25 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 }
 }
 
-private void enableStorageLimitIfPossible(Cuboid cuboid, 
Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
+private void enableStorageLimitIfPossible(CubeDesc cubeDesc, Cuboid 
cuboid, Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
 boolean possible = true;
 
+for (TblColRef col : cuboid.getColumns()) {
+RowKeyColDesc rowKeyDesc = cubeDesc.getRowkey().getColDesc(col);
+if (rowKeyDesc != null) {
+String encodingName = rowKeyDesc.getEncodingName();
+
+//for numbers, if it's fixed_length encoding, order is no 
longer preserved
+if (encodingName != null && 
encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && //
+(col.getType().isNumberFamily() || 
col.getType().isIntegerFamily())) {
+possible = false;
+logger.info("Storage limit push down is impossible because 
integer/number is encoded as fixedlength");
+}
+} else {
+logger.warn("RowKeyColDesc for {} does not exist", col);
+}
+}
+
 boolean goodFilter = filter == null || 
TupleFilter.isEvaluableRecursively(filter);
 if (!goodFilter) {
 possible = false;

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b28fe630
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b28fe630
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b28fe630

Branch: refs/heads/yang21-hbase1.x
Commit: b28fe630bea9d695f0c970ed8024c5327fa95160
Parents: 61335d9
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:16:37 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-   

kylin git commit: KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers in the rowkey

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21 c23205d93 -> 61335d9ce


KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding 
for integers in the rowkey


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61335d9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61335d9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61335d9c

Branch: refs/heads/yang21
Commit: 61335d9ce8ff193bd22e881855d0c68e48896ca9
Parents: c23205d
Author: Hongbin Ma 
Authored: Mon Nov 14 09:12:51 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:12:51 2016 +0800

--
 .../gtrecord/GTCubeStorageQueryBase.java| 22 ++--
 1 file changed, 20 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61335d9c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index b51af59..d648469 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -34,7 +34,9 @@ import org.apache.kylin.cube.RawQueryLastHacker;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -120,7 +122,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 //set whether to aggr at storage
 context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, 
groupsD, singleValuesD));
 // set limit push down
-enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, 
groupsD, filter, sqlDigest.aggregations, context);
+enableStorageLimitIfPossible(cubeInstance.getDescriptor(), cuboid, 
groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, 
context);
 context.setFinalPushDownLimit(cubeInstance);
 // set cautious threshold to prevent out of memory
 setThresholdIfNecessary(dimensionsD, metrics, context);
@@ -412,9 +414,25 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 }
 }
 
-private void enableStorageLimitIfPossible(Cuboid cuboid, 
Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
+private void enableStorageLimitIfPossible(CubeDesc cubeDesc, Cuboid 
cuboid, Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
 boolean possible = true;
 
+for (TblColRef col : cuboid.getColumns()) {
+RowKeyColDesc rowKeyDesc = cubeDesc.getRowkey().getColDesc(col);
+if (rowKeyDesc != null) {
+String encodingName = rowKeyDesc.getEncodingName();
+
+//for numbers, if it's fixed_length encoding, order is no 
longer preserved
+if (encodingName != null && 
encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && //
+(col.getType().isNumberFamily() || 
col.getType().isIntegerFamily())) {
+possible = false;
+logger.info("Storage limit push down is impossible because 
integer/number is encoded as fixedlength");
+}
+} else {
+logger.warn("RowKeyColDesc for {} does not exist", col);
+}
+}
+
 boolean goodFilter = filter == null || 
TupleFilter.isEvaluableRecursively(filter);
 if (!goodFilter) {
 possible = false;

kylin git commit: minor, fix sample cube metadata file rename

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master f1042bf01 -> b6d8e8849


minor, fix sample cube metadata file rename

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6d8e884
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6d8e884
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6d8e884

Branch: refs/heads/master
Commit: b6d8e8849751b12706669069c83a4bedddfc9be0
Parents: f1042bf
Author: Roger Shi 
Authored: Sun Nov 13 22:45:44 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:56:50 2016 +0800

--
 build/bin/sample.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d8e884/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index cc731cf..c1a7bcc 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -52,8 +52,8 @@ fi
 mkdir -p ${KYLIN_HOME}/sample_cube/metadata
 cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
 
-sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
-sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

[1/4] kylin git commit: minor, fix sample cube metadata file rename [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 c15167ee4 -> 5f867e218 (forced update)


minor, fix sample cube metadata file rename

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c23205d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c23205d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c23205d9

Branch: refs/heads/yang21-cdh5.7
Commit: c23205d9394d3149db5771779b54bfec346e60c0
Parents: 29ac66b
Author: Roger Shi 
Authored: Sun Nov 13 22:45:44 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:47:49 2016 +0800

--
 build/bin/sample.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c23205d9/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index cc731cf..c1a7bcc 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -52,8 +52,8 @@ fi
 mkdir -p ${KYLIN_HOME}/sample_cube/metadata
 cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
 
-sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
-sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/130c26df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/130c26df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/130c26df

Branch: refs/heads/yang21-cdh5.7
Commit: 130c26df9da803afa5cace919c6afe673b01f2f5
Parents: c23205d
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:48:46 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
- 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f867e21
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f867e21
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f867e21

Branch: refs/heads/yang21-cdh5.7
Commit: 5f867e218e8d047f111b8b5e3ba6b601116ac361
Parents: 130c26d
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:50:23 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f867e21/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f867e21/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f867e21/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f867e21/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/3] kylin git commit: minor, fix sample cube metadata file rename [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 1e0a6885c -> 130c26df9 (forced update)


minor, fix sample cube metadata file rename

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c23205d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c23205d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c23205d9

Branch: refs/heads/yang21-hbase1.x
Commit: c23205d9394d3149db5771779b54bfec346e60c0
Parents: 29ac66b
Author: Roger Shi 
Authored: Sun Nov 13 22:45:44 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:47:49 2016 +0800

--
 build/bin/sample.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c23205d9/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index cc731cf..c1a7bcc 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -52,8 +52,8 @@ fi
 mkdir -p ${KYLIN_HOME}/sample_cube/metadata
 cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
 
-sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
-sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

kylin git commit: minor, fix sample cube metadata file rename

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21 29ac66bb1 -> c23205d93


minor, fix sample cube metadata file rename

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c23205d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c23205d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c23205d9

Branch: refs/heads/yang21
Commit: c23205d9394d3149db5771779b54bfec346e60c0
Parents: 29ac66b
Author: Roger Shi 
Authored: Sun Nov 13 22:45:44 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:47:49 2016 +0800

--
 build/bin/sample.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c23205d9/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index cc731cf..c1a7bcc 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -52,8 +52,8 @@ fi
 mkdir -p ${KYLIN_HOME}/sample_cube/metadata
 cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
 
-sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
-sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[2/5] kylin git commit: minor, refine sample cube metadata

[mahongbin]

minor, refine sample cube metadata

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29ac66bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29ac66bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29ac66bb

Branch: refs/heads/yang21-cdh5.7
Commit: 29ac66bb171bb2cab4aceff0de24bd985114fa82
Parents: 07bd53b
Author: Roger Shi 
Authored: Sun Nov 13 17:31:21 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 13 19:10:22 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../template/cube/kylin_sales_cube.json |   2 +-
 .../template/cube_desc/kylin_sales_cube.json| 199 +++
 .../cube_desc/kylin_sales_cube_desc.json| 199 ---
 4 files changed, 201 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 11d995c..973ed17 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -237,7 +237,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.error("Cannot found kafka dependency jars: " + e);
+logger.debug("Cannot found kafka dependency jars: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube/kylin_sales_cube.json 
b/examples/sample_cube/template/cube/kylin_sales_cube.json
index 86042ba..3d6d299 100644
--- a/examples/sample_cube/template/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube/kylin_sales_cube.json
@@ -4,7 +4,7 @@
   "last_modified" : 0,
   "name" : "kylin_sales_cube",
   "owner" : null,
-  "descriptor" : "kylin_sales_cube_desc",
+  "descriptor" : "kylin_sales_cube",
   "status" : "DISABLED",
   "segments" : [ ],
   "create_time" : null,

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
new file mode 100644
index 000..eb17922
--- /dev/null
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -0,0 +1,199 @@
+{
+  "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
+  "name" : "kylin_sales_cube",
+  "model_name" : "kylin_sales_model",
+  "description" : null,
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "CAL_DT",
+"table" : "DEFAULT.KYLIN_CAL_DT",
+"column" : "{FK}",
+"derived" : [ "WEEK_BEG_DT" ]
+  }, {
+"name" : "CATEGORY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "{FK}",
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "META_CATEG_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL2_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL3_NAME",
+"derived" : null
+  }, {
+"name" : "LSTG_FORMAT_NAME",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "LSTG_FORMAT_NAME",
+"derived" : null
+  }, {
+"name" : "USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "USER_ID",
+"derived" : null
+  }, {
+"name" : "REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "REGION",
+"derived" : null
+  } ],
+  "measures" : [ {
+"name" : "GMV_SUM",
+"function" : {
+  "expression" : "SUM",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : null
+  },
+  "returntype" : "decimal(19,4)"
+},
+"dependent_measure_ref" : null
+  }, {
+"name" : "GMV_MIN",
+"function" : {
+  

[5/5] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c15167ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c15167ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c15167ee

Branch: refs/heads/yang21-cdh5.7
Commit: c15167ee49d30dea6750d89f2f9bd647c3bdbade
Parents: 1e0a688
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Sun Nov 13 21:40:20 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c15167ee/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/c15167ee/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/c15167ee/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c15167ee/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[1/5] kylin git commit: set ResourceStore verbose logs as Trace level [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 cde86bef3 -> c15167ee4 (forced update)


set ResourceStore verbose logs as Trace level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07bd53b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07bd53b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07bd53b8

Branch: refs/heads/yang21-cdh5.7
Commit: 07bd53b8e63565ccb0b720f7395a6e9127a24946
Parents: ac3f3b6
Author: Hongbin Ma 
Authored: Sun Nov 13 17:04:27 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 17:04:27 2016 +0800

--
 .../java/org/apache/kylin/common/persistence/ResourceStore.java  | 4 ++--
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07bd53b8/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 532e7aa..466d9d2 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -204,7 +204,7 @@ abstract public class ResourceStore {
  */
 final public void putResource(String resPath, InputStream content, long 
ts) throws IOException {
 resPath = norm(resPath);
-logger.debug("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 putResourceImpl(resPath, content, ts);
 }
 
@@ -255,7 +255,7 @@ abstract public class ResourceStore {
  * delete a resource, does nothing on a folder
  */
 final public void deleteResource(String resPath) throws IOException {
-logger.debug("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 deleteResourceImpl(norm(resPath));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/07bd53b8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index e2f3661..8deb811 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -259,7 +259,7 @@ public class HBaseResourceStore extends ResourceStore {
 Put put = buildPut(resPath, newTS, row, content, table);
 
 boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, 
put);
-logger.debug("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
+logger.trace("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
 if (!ok) {
 long real = getResourceTimestampImpl(resPath);
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e0a6885
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e0a6885
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e0a6885

Branch: refs/heads/yang21-cdh5.7
Commit: 1e0a6885ceb5399dbe614ab1930625b4d26a7a34
Parents: 29ac66b
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 21:38:43 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
- 

[1/4] kylin git commit: set ResourceStore verbose logs as Trace level [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x f6735a811 -> 1e0a6885c (forced update)


set ResourceStore verbose logs as Trace level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07bd53b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07bd53b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07bd53b8

Branch: refs/heads/yang21-hbase1.x
Commit: 07bd53b8e63565ccb0b720f7395a6e9127a24946
Parents: ac3f3b6
Author: Hongbin Ma 
Authored: Sun Nov 13 17:04:27 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 17:04:27 2016 +0800

--
 .../java/org/apache/kylin/common/persistence/ResourceStore.java  | 4 ++--
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07bd53b8/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 532e7aa..466d9d2 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -204,7 +204,7 @@ abstract public class ResourceStore {
  */
 final public void putResource(String resPath, InputStream content, long 
ts) throws IOException {
 resPath = norm(resPath);
-logger.debug("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 putResourceImpl(resPath, content, ts);
 }
 
@@ -255,7 +255,7 @@ abstract public class ResourceStore {
  * delete a resource, does nothing on a folder
  */
 final public void deleteResource(String resPath) throws IOException {
-logger.debug("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 deleteResourceImpl(norm(resPath));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/07bd53b8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index e2f3661..8deb811 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -259,7 +259,7 @@ public class HBaseResourceStore extends ResourceStore {
 Put put = buildPut(resPath, newTS, row, content, table);
 
 boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, 
put);
-logger.debug("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
+logger.trace("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
 if (!ok) {
 long real = getResourceTimestampImpl(resPath);
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);

[2/4] kylin git commit: minor, refine sample cube metadata

[mahongbin]

minor, refine sample cube metadata

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29ac66bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29ac66bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29ac66bb

Branch: refs/heads/yang21-hbase1.x
Commit: 29ac66bb171bb2cab4aceff0de24bd985114fa82
Parents: 07bd53b
Author: Roger Shi 
Authored: Sun Nov 13 17:31:21 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 13 19:10:22 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../template/cube/kylin_sales_cube.json |   2 +-
 .../template/cube_desc/kylin_sales_cube.json| 199 +++
 .../cube_desc/kylin_sales_cube_desc.json| 199 ---
 4 files changed, 201 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 11d995c..973ed17 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -237,7 +237,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.error("Cannot found kafka dependency jars: " + e);
+logger.debug("Cannot found kafka dependency jars: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube/kylin_sales_cube.json 
b/examples/sample_cube/template/cube/kylin_sales_cube.json
index 86042ba..3d6d299 100644
--- a/examples/sample_cube/template/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube/kylin_sales_cube.json
@@ -4,7 +4,7 @@
   "last_modified" : 0,
   "name" : "kylin_sales_cube",
   "owner" : null,
-  "descriptor" : "kylin_sales_cube_desc",
+  "descriptor" : "kylin_sales_cube",
   "status" : "DISABLED",
   "segments" : [ ],
   "create_time" : null,

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
new file mode 100644
index 000..eb17922
--- /dev/null
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -0,0 +1,199 @@
+{
+  "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
+  "name" : "kylin_sales_cube",
+  "model_name" : "kylin_sales_model",
+  "description" : null,
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "CAL_DT",
+"table" : "DEFAULT.KYLIN_CAL_DT",
+"column" : "{FK}",
+"derived" : [ "WEEK_BEG_DT" ]
+  }, {
+"name" : "CATEGORY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "{FK}",
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "META_CATEG_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL2_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL3_NAME",
+"derived" : null
+  }, {
+"name" : "LSTG_FORMAT_NAME",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "LSTG_FORMAT_NAME",
+"derived" : null
+  }, {
+"name" : "USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "USER_ID",
+"derived" : null
+  }, {
+"name" : "REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "REGION",
+"derived" : null
+  } ],
+  "measures" : [ {
+"name" : "GMV_SUM",
+"function" : {
+  "expression" : "SUM",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : null
+  },
+  "returntype" : "decimal(19,4)"
+},
+"dependent_measure_ref" : null
+  }, {
+"name" : "GMV_MIN",
+"function" : {
+  

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e0a6885
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e0a6885
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e0a6885

Branch: refs/heads/yang21-hbase1.x
Commit: 1e0a6885ceb5399dbe614ab1930625b4d26a7a34
Parents: 29ac66b
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 21:38:43 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-   

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

kylin git commit: set ResourceStore verbose logs as Trace level

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master a1847b90d -> b7600f87c


set ResourceStore verbose logs as Trace level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b7600f87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b7600f87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b7600f87

Branch: refs/heads/master
Commit: b7600f87c061e4979e9fdde046a0e8606ffba331
Parents: a1847b9
Author: Hongbin Ma 
Authored: Sun Nov 13 17:02:21 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 17:02:21 2016 +0800

--
 .../java/org/apache/kylin/common/persistence/ResourceStore.java  | 4 ++--
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b7600f87/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 532e7aa..466d9d2 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -204,7 +204,7 @@ abstract public class ResourceStore {
  */
 final public void putResource(String resPath, InputStream content, long 
ts) throws IOException {
 resPath = norm(resPath);
-logger.debug("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 putResourceImpl(resPath, content, ts);
 }
 
@@ -255,7 +255,7 @@ abstract public class ResourceStore {
  * delete a resource, does nothing on a folder
  */
 final public void deleteResource(String resPath) throws IOException {
-logger.debug("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 deleteResourceImpl(norm(resPath));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7600f87/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 3fd6426..82cc93c 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -259,7 +259,7 @@ public class HBaseResourceStore extends ResourceStore {
 Put put = buildPut(resPath, newTS, row, content, table);
 
 boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, 
put);
-logger.debug("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
+logger.trace("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
 if (!ok) {
 long real = getResourceTimestampImpl(resPath);
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);

[5/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6735a81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6735a81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6735a81

Branch: refs/heads/yang21-cdh5.7
Commit: f6735a811f0f0f57004e7b75e9ed312da872de67
Parents: ac3f3b6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 15:37:05 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6735a81/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
- 

[6/6] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cde86bef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cde86bef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cde86bef

Branch: refs/heads/yang21-cdh5.7
Commit: cde86bef3323d9000e38ee19d31543b3240f44b0
Parents: f6735a8
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Sun Nov 13 15:38:49 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cde86bef/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/cde86bef/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/cde86bef/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/cde86bef/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[3/6] kylin git commit: minor, remove unnecessary check in BaseCuboidMapperBase

[mahongbin]

minor, remove unnecessary check in BaseCuboidMapperBase


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac3f3b62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac3f3b62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac3f3b62

Branch: refs/heads/yang21-cdh5.7
Commit: ac3f3b621f74c8fb224b67ad32ac38aa426d797e
Parents: d34af64
Author: Yang Li 
Authored: Sun Nov 13 13:12:11 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 13 13:12:11 2016 +0800

--
 .../kylin/cube/model/CubeJoinedFlatTableEnrich.java  | 11 ---
 .../kylin/engine/mr/steps/BaseCuboidMapperBase.java  |  2 --
 2 files changed, 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac3f3b62/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index 8af2297..979af76 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -20,7 +20,6 @@ package org.apache.kylin.cube.model;
 
 import java.util.List;
 
-import org.apache.kylin.common.util.BytesSplitter;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
@@ -81,16 +80,6 @@ public class CubeJoinedFlatTableEnrich implements 
IJoinedFlatTableDesc {
 }
 }
 
-// sanity check the input record (in bytes) matches what's expected
-public void sanityCheck(BytesSplitter bytesSplitter) {
-int columnCount = flatDesc.getAllColumns().size();
-if (columnCount != bytesSplitter.getBufferSize()) {
-throw new IllegalArgumentException("Expect " + columnCount + " 
columns, but see " + bytesSplitter.getBufferSize() + " -- " + bytesSplitter);
-}
-
-// TODO: check data types here
-}
-
 public CubeDesc getCubeDesc() {
 return cubeDesc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac3f3b62/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 0649a0c..dd0a031 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -195,8 +195,6 @@ abstract public class BaseCuboidMapperBase 
extends KylinMapper

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6735a81/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6735a81/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[2/6] kylin git commit: minor, check-env.sh enhance

[mahongbin]

minor, check-env.sh enhance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34af64f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34af64f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34af64f

Branch: refs/heads/yang21-cdh5.7
Commit: d34af64f654a31236160ab298a16e08a08b8e00b
Parents: f5e6198
Author: Yang Li 
Authored: Sat Nov 12 09:21:13 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 12 09:21:13 2016 +0800

--
 build/bin/metastore.sh | 2 +-
 build/bin/sample.sh| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34af64f/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 72afbae..d045ca6 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -26,7 +26,7 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
+source ${dir}/check-env.sh "if-not-yet"
 
 if [ "$1" == "backup" ]
 then

http://git-wip-us.apache.org/repos/asf/kylin/blob/d34af64f/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index e420df7..cc731cf 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -19,7 +19,7 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
+source ${dir}/check-env.sh "if-not-yet"
 job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
 
 echo "Going to create sample tables in hive..."

[1/6] kylin git commit: minor, mr code clean up [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 a084a869a -> cde86bef3 (forced update)


minor, mr code clean up


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f5e61980
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f5e61980
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f5e61980

Branch: refs/heads/yang21-cdh5.7
Commit: f5e619802c54aaca03a1771c9a38af33f6adf50c
Parents: b423384
Author: Yang Li 
Authored: Fri Nov 11 22:51:23 2016 +0800
Committer: Yang Li 
Committed: Fri Nov 11 22:51:23 2016 +0800

--
 .../main/java/org/apache/kylin/engine/mr/KylinMapper.java| 6 --
 .../main/java/org/apache/kylin/engine/mr/KylinReducer.java   | 6 --
 .../java/org/apache/kylin/engine/mr/steps/CuboidJob.java | 8 +++-
 .../org/apache/kylin/engine/mr/steps/InMemCuboidJob.java | 3 ++-
 4 files changed, 13 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index a527b3d..a01f7a2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-abstract public class KylinMapper extends 
Mapper {
+public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
@@ -54,7 +54,9 @@ abstract public class KylinMapper extends Mapp
 }
 }
 
-abstract protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, 
InterruptedException;
+protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, InterruptedException {
+super.map(key, value, context);
+}
 
 @Override
 final protected void cleanup(Mapper.Context context) throws IOException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index 2987032..2b63ce0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-abstract public class KylinReducer extends 
Reducer {
+public class KylinReducer extends 
Reducer {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinReducer.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
@@ -53,7 +53,9 @@ abstract public class KylinReducer extends Red
 }
 }
 
-abstract protected void doReduce(KEYIN key, Iterable values, 
Reducer.Context context) throws IOException, 
InterruptedException;
+protected void doReduce(KEYIN key, Iterable values, 
Reducer.Context context) throws IOException, 
InterruptedException {
+super.reduce(key, values, context);
+}
 
 @Override
 final protected void cleanup(Reducer.Context context) throws IOException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index 6b0c86e..9486e60 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -21,11 +21,11 @@ package 

[5/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6735a81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6735a81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6735a81

Branch: refs/heads/yang21-hbase1.x
Commit: f6735a811f0f0f57004e7b75e9ed312da872de67
Parents: ac3f3b6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 15:37:05 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6735a81/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-   

[2/5] kylin git commit: minor, check-env.sh enhance

[mahongbin]

minor, check-env.sh enhance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34af64f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34af64f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34af64f

Branch: refs/heads/yang21-hbase1.x
Commit: d34af64f654a31236160ab298a16e08a08b8e00b
Parents: f5e6198
Author: Yang Li 
Authored: Sat Nov 12 09:21:13 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 12 09:21:13 2016 +0800

--
 build/bin/metastore.sh | 2 +-
 build/bin/sample.sh| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34af64f/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 72afbae..d045ca6 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -26,7 +26,7 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
+source ${dir}/check-env.sh "if-not-yet"
 
 if [ "$1" == "backup" ]
 then

http://git-wip-us.apache.org/repos/asf/kylin/blob/d34af64f/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index e420df7..cc731cf 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -19,7 +19,7 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-source ${dir}/check-env.sh
+source ${dir}/check-env.sh "if-not-yet"
 job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
 
 echo "Going to create sample tables in hive..."

[1/5] kylin git commit: minor, mr code clean up [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x eb6c3b064 -> f6735a811 (forced update)


minor, mr code clean up


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f5e61980
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f5e61980
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f5e61980

Branch: refs/heads/yang21-hbase1.x
Commit: f5e619802c54aaca03a1771c9a38af33f6adf50c
Parents: b423384
Author: Yang Li 
Authored: Fri Nov 11 22:51:23 2016 +0800
Committer: Yang Li 
Committed: Fri Nov 11 22:51:23 2016 +0800

--
 .../main/java/org/apache/kylin/engine/mr/KylinMapper.java| 6 --
 .../main/java/org/apache/kylin/engine/mr/KylinReducer.java   | 6 --
 .../java/org/apache/kylin/engine/mr/steps/CuboidJob.java | 8 +++-
 .../org/apache/kylin/engine/mr/steps/InMemCuboidJob.java | 3 ++-
 4 files changed, 13 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index a527b3d..a01f7a2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-abstract public class KylinMapper extends 
Mapper {
+public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
@@ -54,7 +54,9 @@ abstract public class KylinMapper extends Mapp
 }
 }
 
-abstract protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, 
InterruptedException;
+protected void doMap(KEYIN key, VALUEIN value, Mapper.Context context) throws IOException, InterruptedException {
+super.map(key, value, context);
+}
 
 @Override
 final protected void cleanup(Mapper.Context context) throws IOException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index 2987032..2b63ce0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  */
-abstract public class KylinReducer extends 
Reducer {
+public class KylinReducer extends 
Reducer {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinReducer.class);
 
 protected void bindCurrentConfiguration(Configuration conf) {
@@ -53,7 +53,9 @@ abstract public class KylinReducer extends Red
 }
 }
 
-abstract protected void doReduce(KEYIN key, Iterable values, 
Reducer.Context context) throws IOException, 
InterruptedException;
+protected void doReduce(KEYIN key, Iterable values, 
Reducer.Context context) throws IOException, 
InterruptedException {
+super.reduce(key, values, context);
+}
 
 @Override
 final protected void cleanup(Reducer.Context context) throws IOException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f5e61980/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index 6b0c86e..9486e60 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -21,11 +21,11 @@ 

[3/5] kylin git commit: minor, remove unnecessary check in BaseCuboidMapperBase

[mahongbin]

minor, remove unnecessary check in BaseCuboidMapperBase


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac3f3b62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac3f3b62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac3f3b62

Branch: refs/heads/yang21-hbase1.x
Commit: ac3f3b621f74c8fb224b67ad32ac38aa426d797e
Parents: d34af64
Author: Yang Li 
Authored: Sun Nov 13 13:12:11 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 13 13:12:11 2016 +0800

--
 .../kylin/cube/model/CubeJoinedFlatTableEnrich.java  | 11 ---
 .../kylin/engine/mr/steps/BaseCuboidMapperBase.java  |  2 --
 2 files changed, 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac3f3b62/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index 8af2297..979af76 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -20,7 +20,6 @@ package org.apache.kylin.cube.model;
 
 import java.util.List;
 
-import org.apache.kylin.common.util.BytesSplitter;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
@@ -81,16 +80,6 @@ public class CubeJoinedFlatTableEnrich implements 
IJoinedFlatTableDesc {
 }
 }
 
-// sanity check the input record (in bytes) matches what's expected
-public void sanityCheck(BytesSplitter bytesSplitter) {
-int columnCount = flatDesc.getAllColumns().size();
-if (columnCount != bytesSplitter.getBufferSize()) {
-throw new IllegalArgumentException("Expect " + columnCount + " 
columns, but see " + bytesSplitter.getBufferSize() + " -- " + bytesSplitter);
-}
-
-// TODO: check data types here
-}
-
 public CubeDesc getCubeDesc() {
 return cubeDesc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac3f3b62/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 0649a0c..dd0a031 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -195,8 +195,6 @@ abstract public class BaseCuboidMapperBase 
extends KylinMapper

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6735a81/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6735a81/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

kylin git commit: KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan NEW segment in cube when exception is met

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 0fb0a75bd -> a6e8c35da


KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan 
NEW segment in cube when exception is met


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a6e8c35d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a6e8c35d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a6e8c35d

Branch: refs/heads/master
Commit: a6e8c35da8e2c6d944dce4c7d261a541c312b7f6
Parents: 0fb0a75
Author: Hongbin Ma 
Authored: Fri Nov 11 16:32:00 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:55:57 2016 +0800

--
 .../apache/kylin/rest/service/JobService.java   | 58 ++--
 1 file changed, 40 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e8c35d/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 856c754..a1aadab 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -270,25 +270,47 @@ public class JobService extends BasicService implements 
InitializingBean {
 
 checkCubeDescSignature(cube);
 DefaultChainedExecutable job;
-if (buildType == CubeBuildTypeEnum.BUILD) {
-ISource source = SourceFactory.tableSource(cube);
-SourcePartition sourcePartition = new SourcePartition(startDate, 
endDate, startOffset, endOffset, sourcePartitionOffsetStart, 
sourcePartitionOffsetEnd);
-sourcePartition = source.parsePartitionBeforeBuild(cube, 
sourcePartition);
-CubeSegment newSeg = getCubeManager().appendSegment(cube, 
sourcePartition);
-lockSegment(newSeg.getUuid());
-job = EngineFactory.createBatchCubingJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.MERGE) {
-CubeSegment newSeg = getCubeManager().mergeSegments(cube, 
startDate, endDate, startOffset, endOffset, force);
-lockSegment(newSeg.getUuid());
-job = EngineFactory.createBatchMergeJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.REFRESH) {
-CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, 
startDate, endDate, startOffset, endOffset);
-lockSegment(refreshSeg.getUuid());
-job = EngineFactory.createBatchCubingJob(refreshSeg, submitter);
-} else {
-throw new JobException("invalid build type:" + buildType);
+
+CubeSegment newSeg = null;
+try {
+if (buildType == CubeBuildTypeEnum.BUILD) {
+ISource source = SourceFactory.tableSource(cube);
+SourcePartition sourcePartition = new 
SourcePartition(startDate, endDate, startOffset, endOffset, 
sourcePartitionOffsetStart, sourcePartitionOffsetEnd);
+sourcePartition = source.parsePartitionBeforeBuild(cube, 
sourcePartition);
+newSeg = getCubeManager().appendSegment(cube, sourcePartition);
+lockSegment(newSeg.getUuid());
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.MERGE) {
+newSeg = getCubeManager().mergeSegments(cube, startDate, 
endDate, startOffset, endOffset, force);
+lockSegment(newSeg.getUuid());
+job = EngineFactory.createBatchMergeJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.REFRESH) {
+newSeg = getCubeManager().refreshSegment(cube, startDate, 
endDate, startOffset, endOffset);
+lockSegment(newSeg.getUuid());
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else {
+throw new JobException("invalid build type:" + buildType);
+}
+
+getExecutableManager().addJob(job);
+
+} catch (Exception e) {
+if (newSeg != null) {
+logger.error("Job submission might failed for NEW segment {}, 
will clean the NEW segment from cube", newSeg.getName());
+try {
+// Remove this segments
+CubeUpdate cubeBuilder = new CubeUpdate(cube);
+cubeBuilder.setToRemoveSegs(newSeg);
+getCubeManager().updateCube(cubeBuilder);
+} catch (Exception ee) {
+// swallow the exception
+  

[1/6] kylin git commit: minor, JobService show possible jobs in the future [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 33543a90b -> a084a869a (forced update)


minor, JobService show possible jobs in the future


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f03350
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f03350
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f03350

Branch: refs/heads/yang21-cdh5.7
Commit: 83f03350629f5c0963378c5db348bbaebd8e9aa2
Parents: 6ba39df
Author: Li Yang 
Authored: Fri Nov 11 16:08:30 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 11 16:08:51 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/JobService.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f03350/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 5c704ba..2e547c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -95,9 +95,8 @@ public class JobService extends BasicService {
 public List listAllJobs(final String cubeName, final String 
projectName, final List statusList, final JobTimeFilterEnum 
timeFilter) {
 Calendar calendar = Calendar.getInstance();
 calendar.setTime(new Date());
-long currentTimeMillis = calendar.getTimeInMillis();
 long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
-return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, currentTimeMillis);
+return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, Long.MAX_VALUE);
 }
 
 @Deprecated

[3/6] kylin git commit: KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan NEW segment in cube when exception is met

[mahongbin]

KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan 
NEW segment in cube when exception is met


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4233844
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4233844
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4233844

Branch: refs/heads/yang21-cdh5.7
Commit: b4233844816a722c3259109804d20a9fb8f5e954
Parents: 2e3c7ca
Author: Hongbin Ma 
Authored: Fri Nov 11 16:32:00 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:32:05 2016 +0800

--
 .../apache/kylin/rest/service/JobService.java   | 43 ++--
 1 file changed, 31 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4233844/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 2e547c7..186e265 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -209,19 +209,38 @@ public class JobService extends BasicService {
 
 DefaultChainedExecutable job;
 
-if (buildType == CubeBuildTypeEnum.BUILD) {
-CubeSegment newSeg = getCubeManager().appendSegment(cube, 
startDate, endDate, startOffset, endOffset);
-job = EngineFactory.createBatchCubingJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.MERGE) {
-CubeSegment newSeg = getCubeManager().mergeSegments(cube, 
startDate, endDate, startOffset, endOffset, force);
-job = EngineFactory.createBatchMergeJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.REFRESH) {
-CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, 
startDate, endDate, startOffset, endOffset);
-job = EngineFactory.createBatchCubingJob(refreshSeg, submitter);
-} else {
-throw new JobException("invalid build type:" + buildType);
+CubeSegment newSeg = null;
+try {
+if (buildType == CubeBuildTypeEnum.BUILD) {
+newSeg = getCubeManager().appendSegment(cube, startDate, 
endDate, startOffset, endOffset);
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.MERGE) {
+newSeg = getCubeManager().mergeSegments(cube, startDate, 
endDate, startOffset, endOffset, force);
+job = EngineFactory.createBatchMergeJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.REFRESH) {
+newSeg = getCubeManager().refreshSegment(cube, startDate, 
endDate, startOffset, endOffset);
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else {
+throw new JobException("invalid build type:" + buildType);
+}
+getExecutableManager().addJob(job);
+
+} catch (Exception e) {
+if (newSeg != null) {
+logger.error("Job submission might failed for NEW segment {}, 
will clean the NEW segment from cube", newSeg.getName());
+try {
+// Remove this segments
+CubeUpdate cubeBuilder = new CubeUpdate(cube);
+cubeBuilder.setToRemoveSegs(newSeg);
+getCubeManager().updateCube(cubeBuilder);
+} catch (Exception ee) {
+// swallow the exception
+logger.error("Clean New segment failed, ignoring it", e);
+}
+}
+throw e;
 }
-getExecutableManager().addJob(job);
+
 JobInstance jobInstance = getSingleJobInstance(job);
 
 accessService.init(jobInstance, null);

[6/6] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[mahongbin]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a084a869
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a084a869
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a084a869

Branch: refs/heads/yang21-cdh5.7
Commit: a084a869ad0726084805169658a0e47e25575960
Parents: eb6c3b0
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:45:15 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a084a869/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/a084a869/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/a084a869/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a084a869/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[2/5] kylin git commit: ignore 'defaultCatalog' in sql

[mahongbin]

ignore 'defaultCatalog' in sql


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e3c7ca4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e3c7ca4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e3c7ca4

Branch: refs/heads/yang21-hbase1.x
Commit: 2e3c7ca43d2e5b6848a45abbffde91774fd4bf7c
Parents: 83f0335
Author: shaofengshi 
Authored: Fri Nov 11 16:12:34 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 11 16:12:34 2016 +0800

--
 .../java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java| 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e3c7ca4/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
index 8d8d971..d8c8845 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
@@ -27,6 +27,7 @@ public class KeywordDefaultDirtyHack implements 
IQueryTransformer {
 // KYLIN-2108, DEFAULT is hive default database, but a sql keyword 
too, needs quote
 sql = sql.replace("DEFAULT.", "\"DEFAULT\".");
 sql = sql.replace("default.", "\"default\".");
+sql = sql.replace("defaultCatalog.", "");
 
 return sql;
 }

[1/5] kylin git commit: minor, JobService show possible jobs in the future [Forced Update!]

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x fb74c6195 -> eb6c3b064 (forced update)


minor, JobService show possible jobs in the future


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f03350
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f03350
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f03350

Branch: refs/heads/yang21-hbase1.x
Commit: 83f03350629f5c0963378c5db348bbaebd8e9aa2
Parents: 6ba39df
Author: Li Yang 
Authored: Fri Nov 11 16:08:30 2016 +0800
Committer: Li Yang 
Committed: Fri Nov 11 16:08:51 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/JobService.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f03350/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 5c704ba..2e547c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -95,9 +95,8 @@ public class JobService extends BasicService {
 public List listAllJobs(final String cubeName, final String 
projectName, final List statusList, final JobTimeFilterEnum 
timeFilter) {
 Calendar calendar = Calendar.getInstance();
 calendar.setTime(new Date());
-long currentTimeMillis = calendar.getTimeInMillis();
 long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
-return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, currentTimeMillis);
+return listCubeJobInstance(cubeName, projectName, statusList, 
timeStartInMillis, Long.MAX_VALUE);
 }
 
 @Deprecated

[5/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb6c3b06
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb6c3b06
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb6c3b06

Branch: refs/heads/yang21-hbase1.x
Commit: eb6c3b06476dafdc551b88b4d956ef6cc1148b40
Parents: b423384
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:43:39 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb6c3b06/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-   

[3/5] kylin git commit: KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan NEW segment in cube when exception is met

[mahongbin]

KYLIN-2176 org.apache.kylin.rest.service.JobService#submitJob will leave orphan 
NEW segment in cube when exception is met


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4233844
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4233844
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4233844

Branch: refs/heads/yang21-hbase1.x
Commit: b4233844816a722c3259109804d20a9fb8f5e954
Parents: 2e3c7ca
Author: Hongbin Ma 
Authored: Fri Nov 11 16:32:00 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 11 16:32:05 2016 +0800

--
 .../apache/kylin/rest/service/JobService.java   | 43 ++--
 1 file changed, 31 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4233844/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 2e547c7..186e265 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -209,19 +209,38 @@ public class JobService extends BasicService {
 
 DefaultChainedExecutable job;
 
-if (buildType == CubeBuildTypeEnum.BUILD) {
-CubeSegment newSeg = getCubeManager().appendSegment(cube, 
startDate, endDate, startOffset, endOffset);
-job = EngineFactory.createBatchCubingJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.MERGE) {
-CubeSegment newSeg = getCubeManager().mergeSegments(cube, 
startDate, endDate, startOffset, endOffset, force);
-job = EngineFactory.createBatchMergeJob(newSeg, submitter);
-} else if (buildType == CubeBuildTypeEnum.REFRESH) {
-CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, 
startDate, endDate, startOffset, endOffset);
-job = EngineFactory.createBatchCubingJob(refreshSeg, submitter);
-} else {
-throw new JobException("invalid build type:" + buildType);
+CubeSegment newSeg = null;
+try {
+if (buildType == CubeBuildTypeEnum.BUILD) {
+newSeg = getCubeManager().appendSegment(cube, startDate, 
endDate, startOffset, endOffset);
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.MERGE) {
+newSeg = getCubeManager().mergeSegments(cube, startDate, 
endDate, startOffset, endOffset, force);
+job = EngineFactory.createBatchMergeJob(newSeg, submitter);
+} else if (buildType == CubeBuildTypeEnum.REFRESH) {
+newSeg = getCubeManager().refreshSegment(cube, startDate, 
endDate, startOffset, endOffset);
+job = EngineFactory.createBatchCubingJob(newSeg, submitter);
+} else {
+throw new JobException("invalid build type:" + buildType);
+}
+getExecutableManager().addJob(job);
+
+} catch (Exception e) {
+if (newSeg != null) {
+logger.error("Job submission might failed for NEW segment {}, 
will clean the NEW segment from cube", newSeg.getName());
+try {
+// Remove this segments
+CubeUpdate cubeBuilder = new CubeUpdate(cube);
+cubeBuilder.setToRemoveSegs(newSeg);
+getCubeManager().updateCube(cubeBuilder);
+} catch (Exception ee) {
+// swallow the exception
+logger.error("Clean New segment failed, ignoring it", e);
+}
+}
+throw e;
 }
-getExecutableManager().addJob(job);
+
 JobInstance jobInstance = getSingleJobInstance(job);
 
 accessService.init(jobInstance, null);

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[mahongbin]

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb74c619/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb74c619/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+