[2/2] kylin git commit: KYLIN-2308 Allow user to set more columnFamily in web

[zhongjian]

KYLIN-2308 Allow user to set more columnFamily in web

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8331d8d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8331d8d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8331d8d0

Branch: refs/heads/master
Commit: 8331d8d0ad37d24bdbfa90d1032bcdd063aa0a59
Parents: a853a7c
Author: kangkaisen 
Authored: Sat Jan 7 15:34:57 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 17:25:28 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 75 +++-
 webapp/app/js/controllers/cubeEdit.js   | 51 ---
 webapp/app/js/controllers/cubeSchema.js | 13 +++
 webapp/app/js/filters/filter.js | 22 +++--
 .../cubeDesigner/advanced_settings.html | 92 +++-
 5 files changed, 194 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 760133a..39d36b0 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -278,11 +278,84 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 $scope.isReuse=!$scope.isReuse;
   }
 
-  $scope.removeDictionaries =  function(arr,element){
+  $scope.removeElement =  function(arr,element){
 var index = arr.indexOf(element);
 if (index > -1) {
   arr.splice(index, 1);
 }
   };
 
+  $scope.newColFamily = function (index) {
+return {
+"name": "F" + index,
+"columns": [
+  {
+"qualifier": "M",
+"measure_refs": []
+  }
+]
+  };
+  };
+
+  $scope.initColumnFamily = function () {
+$scope.cubeMetaFrame.hbase_mapping.column_family = [];
+var normalMeasures = [], distinctCountMeasures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  if (measure.function.expression === 'COUNT_DISTINCT') {
+distinctCountMeasures.push(measure);
+  } else {
+normalMeasures.push(measure);
+  }
+});
+if (normalMeasures.length > 0) {
+  var nmcf = $scope.newColFamily(1);
+  angular.forEach(normalMeasures, function (normalM, index) {
+nmcf.columns[0].measure_refs.push(normalM.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(nmcf);
+}
+
+if (distinctCountMeasures.length > 0) {
+  var dccf = $scope.newColFamily(2);
+  angular.forEach(distinctCountMeasures, function (dcm, index) {
+dccf.columns[0].measure_refs.push(dcm.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(dccf);
+}
+  };
+
+  $scope.getAllMeasureNames = function () {
+var measures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  measures.push(measure.name);
+});
+return measures;
+  };
+
+  $scope.getAssignedMeasureNames = function () {
+var assignedMeasures = [];
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily, index) {
+  angular.forEach(colFamily.columns[0].measure_refs, function (measure, 
index) {
+assignedMeasures.push(measure);
+  });
+});
+return assignedMeasures;
+  };
+
+  if ($scope.getAllMeasureNames().length != 
$scope.getAssignedMeasureNames().length) {
+$scope.initColumnFamily();
+  }
+
+
+  $scope.addColumnFamily = function () {
+var colFamily = 
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 
1);
+$scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
+  };
+
+  $scope.refreshColumnFamily = function (column_familys, index, colFamily) {
+if (column_familys) {
+  column_familys[index] = colFamily;
+}
+  };
+
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index edbb421..da19b22 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -284,25 +284,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return type;
   };
 
-  var ColFamily = function () {
-var index = 1;
-return function () {
-  var newColFamily =
-  {
-"name": "f" + index,
-"columns": [
-  {
-

kylin git commit: Make some properties configable

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2374 24cae5c73 -> 03e3dc928


Make some properties configable

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03e3dc92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03e3dc92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03e3dc92

Branch: refs/heads/KYLIN-2374
Commit: 03e3dc92862ba846b8098e83c79d8f7f32f774a9
Parents: 24cae5c
Author: xiefan46 <958034...@qq.com>
Authored: Fri Jan 20 18:15:56 2017 +0800
Committer: shaofengshi 
Committed: Fri Jan 20 18:19:11 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 10 +++-
 .../test_case_data/sandbox/kylin.properties |  4 +++
 .../storage/hdfs/ITHDFSResourceStoreTest.java   |  1 -
 .../kylin/storage/hdfs/ITLockManagerTest.java   |  1 -
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 21 ---
 .../apache/kylin/storage/hdfs/LockManager.java  | 19 +++---
 .../kylin/storage/hdfs/ZookeeperConfig.java | 27 
 7 files changed, 34 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 0602e9d..a2038aa 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -775,7 +775,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "10.0"));
 }
 
-
 public int getSparkMinPartition() {
 return Integer.valueOf(getOptional("kylin.engine.spark.min-partition", 
"1"));
 }
@@ -925,4 +924,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
 }
 
+//zoo keeper
+public String getZooKeeperHost() {
+return getOptional("kylin.storage-zookeeper.host", "localhost");
+}
+
+public String getZooKeeperPort() {
+return getOptional("kylin.storage-zookeeper.port", "2181");
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 2d3cd79..cc28541 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -177,3 +177,7 @@ 
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
 #kylin.engine.spark-conf.spark.yarn.queue=default
 
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
+
+
+#zoo keeper
+kylin.storage-zookeeper.host=sandbox
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 41bbcc0..27d8a3c 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -24,7 +24,6 @@ import org.apache.kylin.common.persistence.ResourceStoreTest;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
index 56347e4..2b58d30 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -26,7 +26,6 @@ import 

[7/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5feb9e92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5feb9e92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5feb9e92

Branch: refs/heads/master-cdh5.7
Commit: 5feb9e928b561997649f107006052a6a2184d2f7
Parents: 85a1eb3
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Jan 20 18:05:38 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5feb9e92/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5feb9e92/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
--- 

[6/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/5feb9e92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClassOld2)) {
 desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch 

[1/8] kylin git commit: KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter into RoaringBitmapCounter [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master-cdh5.7 28ae20eb5 -> c2fa18a3b (forced update)


KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter 
into RoaringBitmapCounter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38c3e7bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38c3e7bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38c3e7bf

Branch: refs/heads/master-cdh5.7
Commit: 38c3e7bf691ecdfd0f8d42fcc97065a0596be018
Parents: 61833d9
Author: gaodayue 
Authored: Thu Jan 19 15:23:34 2017 +0800
Committer: gaodayue 
Committed: Thu Jan 19 15:43:14 2017 +0800

--
 .../gridtable/AggregationCacheMemSizeTest.java  |  11 +-
 .../metadata/measure/MeasureCodecTest.java  |   5 +-
 .../kylin/measure/bitmap/BitmapAggregator.java  |  25 +---
 .../kylin/measure/bitmap/BitmapCounter.java |  28 +++-
 .../measure/bitmap/BitmapCounterFactory.java|  30 
 .../BitmapIntersectDistinctCountAggFunc.java|  16 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |  17 ++-
 .../kylin/measure/bitmap/BitmapSerializer.java  |   9 +-
 .../measure/bitmap/ImmutableBitmapCounter.java  | 108 --
 .../measure/bitmap/MutableBitmapCounter.java|  60 
 .../measure/bitmap/RoaringBitmapCounter.java| 147 +++
 .../bitmap/RoaringBitmapCounterFactory.java |  47 ++
 .../measure/AggregatorMemEstimateTest.java  |   5 +-
 .../measure/bitmap/BitmapAggregatorTest.java|  56 ++-
 .../kylin/measure/bitmap/BitmapCounterTest.java |  32 +---
 .../measure/bitmap/BitmapSerializerTest.java|   5 +-
 16 files changed, 345 insertions(+), 256 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 63c7672..f749fb4 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -33,7 +33,9 @@ import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
 import org.apache.kylin.measure.basic.DoubleSumAggregator;
 import org.apache.kylin.measure.basic.LongSumAggregator;
 import org.apache.kylin.measure.bitmap.BitmapAggregator;
-import org.apache.kylin.measure.bitmap.MutableBitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounterFactory;
+import org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory;
 import org.apache.kylin.measure.hllc.HLLCAggregator;
 import org.apache.kylin.measure.hllc.HLLCounter;
 import org.github.jamm.MemoryMeter;
@@ -43,13 +45,14 @@ import com.google.common.base.Stopwatch;
 
 public class AggregationCacheMemSizeTest {
 private static final MemoryMeter meter = new MemoryMeter();
-private static final MutableBitmapCounter[] bitmaps = new 
MutableBitmapCounter[5];
+private static final BitmapCounterFactory bitmapFactory = 
RoaringBitmapCounterFactory.INSTANCE;
+private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
 private static final Random random = new Random();
 
 // consider bitmaps with variant cardinality
 static {
 for (int i = 0; i < bitmaps.length; i++) {
-bitmaps[i] = new MutableBitmapCounter();
+bitmaps[i] = bitmapFactory.newBitmap();
 }
 
 final int totalBits = 1_000_000;
@@ -116,7 +119,7 @@ public class AggregationCacheMemSizeTest {
 }
 
 private BitmapAggregator createBitmapAggr(boolean lowCardinality) {
-MutableBitmapCounter counter = new MutableBitmapCounter();
+BitmapCounter counter = bitmapFactory.newBitmap();
 counter.orWith(lowCardinality ? bitmaps[0] : bitmaps[3]);
 
 BitmapAggregator result = new BitmapAggregator();

http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 97c9751..7129a5e 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -25,7 +25,8 @@ import java.nio.ByteBuffer;
 
 import 

[8/8] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2fa18a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2fa18a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2fa18a3

Branch: refs/heads/master-cdh5.7
Commit: c2fa18a3bb88ea3fba06bb005d69013c255b258e
Parents: 5feb9e9
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Fri Jan 20 18:07:44 2017 +0800

--
 .../kylin/engine/mr/steps/MockupMapContext.java |  13 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  29 +-
 tool/pom.xml|  12 +
 10 files changed, 430 insertions(+), 1137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2fa18a3/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index dfbba14..bc9601f 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -44,10 +44,10 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
 
 /**
  * @author yangli9
- * 
+ *
  */
 @SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext implements MapContext{
+public class MockupMapContext implements MapContext {
 
 private Configuration hconf;
 
@@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext{
 return new WrappedMapper().getMapContext(new MockupMapContext(hconf, 
outKV));
 }
 
-public MockupMapContext(Configuration hconf, Object[] outKV){
+public MockupMapContext(Configuration hconf, Object[] outKV) {
 this.hconf = hconf;
 this.outKV = outKV;
 }
@@ -175,6 +175,11 @@ public class MockupMapContext implements MapContext{
 }
 
 @Override
+public boolean userClassesTakesPrecedence() {
+throw new NotImplementedException();
+}
+
+@Override
 public Class> getInputFormatClass() throws 
ClassNotFoundException {
 throw new NotImplementedException();
 }
@@ -318,4 +323,4 @@ public class MockupMapContext implements MapContext{
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2fa18a3/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+
org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec
 
-
 
-hadoop.http.authentication.simple.anonymous.allowed
-true
+hadoop.security.authentication
+simple
 
-
 
-hadoop.proxyuser.falcon.groups
-users
+hadoop.security.authorization
+false
+
+
+hadoop.rpc.protection
+authentication
+
+
+hadoop.security.auth_to_local
+DEFAULT
 
-
 
-hadoop.proxyuser.falcon.hosts
+hadoop.proxyuser.oozie.hosts
 *
 
-
 
-hadoop.proxyuser.hbase.groups
-users
+

[5/8] kylin git commit: minor, fix false positive warning in test output

[liyang]

minor, fix false positive warning in test output


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85a1eb39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85a1eb39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85a1eb39

Branch: refs/heads/master-cdh5.7
Commit: 85a1eb399d8dcd099d4607aa802faa5ccdf4d2bf
Parents: 1e4ae54
Author: Billy Liu 
Authored: Fri Jan 20 08:13:11 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 08:13:11 2017 +0800

--
 .../org/apache/kylin/common/util/AbstractKylinTestCase.java   | 7 ++-
 .../java/org/apache/kylin/dict/lookup/SnapshotManager.java| 6 ++
 .../src/main/java/org/apache/kylin/job/dao/ExecutableDao.java | 6 ++
 .../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 2 +-
 examples/test_case_data/localmeta/cube_desc/ssb.json  | 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube1.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube2.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube3.json| 4 ++--
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 4 ++--
 .../localmeta/cube_desc/test_streaming_table_cube_desc.json   | 2 +-
 examples/test_case_data/localmeta/kylin.properties| 2 +-
 examples/test_case_data/sandbox/kylin.properties  | 2 +-
 kylin-it/pom.xml  | 2 +-
 13 files changed, 33 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 2154c32..4a9804d 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -31,12 +31,17 @@ public abstract class AbstractKylinTestCase {
 public static final String[] SERVICES_WITH_CACHE = { //
 "org.apache.kylin.cube.CubeManager", //
 "org.apache.kylin.cube.CubeDescManager", //
+"org.apache.kylin.dict.lookup.SnapshotManager", //
+"org.apache.kylin.dict.DictionaryManager", //
 "org.apache.kylin.storage.hybrid.HybridManager", //
 "org.apache.kylin.metadata.realization.RealizationRegistry", //
 "org.apache.kylin.metadata.project.ProjectManager", //
 "org.apache.kylin.metadata.MetadataManager", //
+"org.apache.kylin.metadata.cachesync.Broadcaster", //
+"org.apache.kylin.metadata.badquery.BadQueryHistoryManager", //
 "org.apache.kylin.job.impl.threadpool.DistributedScheduler", //
-"org.apache.kylin.job.manager.ExecutableManager", //
+"org.apache.kylin.job.execution.ExecutableManager", //
+"org.apache.kylin.job.dao.ExecutableDao" //
 };
 
 public abstract void createTestMetadata() throws Exception;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 085158a..b45d017 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -66,6 +66,12 @@ public class SnapshotManager {
 return r;
 }
 
+public static void clearCache() {
+synchronized (SERVICE_CACHE) {
+SERVICE_CACHE.clear();
+}
+}
+
 // 

 
 private KylinConfig config;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java 
b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 5cae5ac..96505e6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -64,6 +64,12 @@ public class ExecutableDao {
 return r;
 }
 
+public static void clearCache() {
+  

[3/8] kylin git commit: KYLIN-2411 Kill MR job on pause

[liyang]

KYLIN-2411 Kill MR job on pause


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b49c9e39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b49c9e39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b49c9e39

Branch: refs/heads/master-cdh5.7
Commit: b49c9e3951c29a21e71f179ee77e2295fbc52ba9
Parents: 38c3e7b
Author: shaofengshi 
Authored: Thu Jan 19 11:02:12 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:37:42 2017 +0800

--
 .../org/apache/kylin/job/execution/ExecutableManager.java| 2 +-
 .../apache/kylin/engine/mr/common/MapReduceExecutable.java   | 8 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 466cdad..48cedb5 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -347,7 +347,7 @@ public class ExecutableManager {
 for (AbstractExecutable task : tasks) {
 if (task.getId().compareTo(stepId) >= 0) {
 logger.debug("rollback task : " + task);
-updateJobOutput(task.getId(), ExecutableState.READY, null, 
null);
+updateJobOutput(task.getId(), ExecutableState.READY, 
Maps.newHashMap(), "");
 }
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index f887c4c..6de07ca 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -148,7 +148,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 //boolean useKerberosAuth = 
context.getConfig().isGetJobStatusWithKerberos();
 //HadoopStatusChecker statusChecker = new 
HadoopStatusChecker(restStatusCheckUrl, mrJobId, output, useKerberosAuth);
 JobStepStatusEnum status = JobStepStatusEnum.NEW;
-while (!isDiscarded()) {
+while (!isDiscarded() && !isPaused()) {
 
 JobStepStatusEnum newStatus = 
HadoopJobStatusChecker.checkStatus(job, output);
 if (status == JobStepStatusEnum.KILLED) {
@@ -184,7 +184,11 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 }
 }
 
-return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
+if (isDiscarded()) {
+return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
+} else {
+return new ExecuteResult(ExecuteResult.State.STOPPED, 
output.toString());
+}
 
 } catch (ReflectiveOperationException e) {
 logger.error("error getMapReduceJobClass, class name:" + 
getParam(KEY_MR_JOB), e);

[2/8] kylin git commit: KYLIN-2409 change inmem cubing to single thread by default

[liyang]

KYLIN-2409 change inmem cubing to single thread by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21969753
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21969753
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21969753

Branch: refs/heads/master-cdh5.7
Commit: 21969753cc66efa22bc9fc933af46346e9846631
Parents: b49c9e3
Author: shaofengshi 
Authored: Thu Jan 19 11:52:17 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:37:42 2017 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 6 +-
 .../kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java| 2 +-
 .../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 9 +
 .../org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java | 2 ++
 4 files changed, 17 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d6774ff..74903d5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -300,6 +300,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
 }
 
+public int getCubeAlgorithmInMemConcurrentThreads() {
+return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-concurrent-threads", 
"1"));
+}
+
 public boolean isIgnoreCubeSignatureInconsistency() {
 return 
Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", 
"false"));
 }
@@ -744,7 +748,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getYarnStatusCheckIntervalSeconds() {
-return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"60"));
+return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"10"));
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 651203a..c7a4a05 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -46,7 +46,7 @@ abstract public class AbstractInMemCubeBuilder {
 final protected CubeDesc cubeDesc;
 final protected Map dictionaryMap;
 
-protected int taskThreadCount = 4;
+protected int taskThreadCount = 1;
 protected int reserveMemoryMB = 100;
 
 public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc 
flatDesc, Map dictionaryMap) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 55c04c6..9158aa3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -36,6 +36,7 @@ import java.util.Map.Entry;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.common.util.Pair;
@@ -178,11 +179,16 @@ public class GTAggregateScanner implements IGTScanner {
 final List dumps;
 final int keyLength;
 final boolean[] compareMask;
+boolean compareAll = true;
 final BufferedMeasureCodec measureCodec;
 
 final Comparator bytesComparator = new Comparator() {
 @Override
 public int compare(byte[] o1, byte[] o2) {
+   

[4/8] kylin git commit: KYLIN-2375 change default ehcache size to 256M

[liyang]

KYLIN-2375 change default ehcache size to 256M


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e4ae540
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e4ae540
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e4ae540

Branch: refs/heads/master-cdh5.7
Commit: 1e4ae540bf194c7e67bb622c2eb0cd0ff1a6657c
Parents: 2196975
Author: shaofengshi 
Authored: Thu Jan 19 17:44:19 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:44:19 2017 +0800

--
 server/src/main/resources/ehcache-test.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e4ae540/server/src/main/resources/ehcache-test.xml
--
diff --git a/server/src/main/resources/ehcache-test.xml 
b/server/src/main/resources/ehcache-test.xml
index eb39774..bffe27a 100644
--- a/server/src/main/resources/ehcache-test.xml
+++ b/server/src/main/resources/ehcache-test.xml
@@ -12,7 +12,7 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
->
+>
 

[3/7] kylin git commit: KYLIN-2411 Kill MR job on pause

[liyang]

KYLIN-2411 Kill MR job on pause


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b49c9e39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b49c9e39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b49c9e39

Branch: refs/heads/master-hbase1.x
Commit: b49c9e3951c29a21e71f179ee77e2295fbc52ba9
Parents: 38c3e7b
Author: shaofengshi 
Authored: Thu Jan 19 11:02:12 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:37:42 2017 +0800

--
 .../org/apache/kylin/job/execution/ExecutableManager.java| 2 +-
 .../apache/kylin/engine/mr/common/MapReduceExecutable.java   | 8 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 466cdad..48cedb5 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -347,7 +347,7 @@ public class ExecutableManager {
 for (AbstractExecutable task : tasks) {
 if (task.getId().compareTo(stepId) >= 0) {
 logger.debug("rollback task : " + task);
-updateJobOutput(task.getId(), ExecutableState.READY, null, 
null);
+updateJobOutput(task.getId(), ExecutableState.READY, 
Maps.newHashMap(), "");
 }
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index f887c4c..6de07ca 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -148,7 +148,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 //boolean useKerberosAuth = 
context.getConfig().isGetJobStatusWithKerberos();
 //HadoopStatusChecker statusChecker = new 
HadoopStatusChecker(restStatusCheckUrl, mrJobId, output, useKerberosAuth);
 JobStepStatusEnum status = JobStepStatusEnum.NEW;
-while (!isDiscarded()) {
+while (!isDiscarded() && !isPaused()) {
 
 JobStepStatusEnum newStatus = 
HadoopJobStatusChecker.checkStatus(job, output);
 if (status == JobStepStatusEnum.KILLED) {
@@ -184,7 +184,11 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 }
 }
 
-return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
+if (isDiscarded()) {
+return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
+} else {
+return new ExecuteResult(ExecuteResult.State.STOPPED, 
output.toString());
+}
 
 } catch (ReflectiveOperationException e) {
 logger.error("error getMapReduceJobClass, class name:" + 
getParam(KEY_MR_JOB), e);

[7/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5feb9e92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5feb9e92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5feb9e92

Branch: refs/heads/master-hbase1.x
Commit: 5feb9e928b561997649f107006052a6a2184d2f7
Parents: 85a1eb3
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Jan 20 18:05:38 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5feb9e92/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5feb9e92/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
--- 

[5/7] kylin git commit: minor, fix false positive warning in test output

[liyang]

minor, fix false positive warning in test output


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85a1eb39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85a1eb39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85a1eb39

Branch: refs/heads/master-hbase1.x
Commit: 85a1eb399d8dcd099d4607aa802faa5ccdf4d2bf
Parents: 1e4ae54
Author: Billy Liu 
Authored: Fri Jan 20 08:13:11 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 08:13:11 2017 +0800

--
 .../org/apache/kylin/common/util/AbstractKylinTestCase.java   | 7 ++-
 .../java/org/apache/kylin/dict/lookup/SnapshotManager.java| 6 ++
 .../src/main/java/org/apache/kylin/job/dao/ExecutableDao.java | 6 ++
 .../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 2 +-
 examples/test_case_data/localmeta/cube_desc/ssb.json  | 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube1.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube2.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube3.json| 4 ++--
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 4 ++--
 .../localmeta/cube_desc/test_streaming_table_cube_desc.json   | 2 +-
 examples/test_case_data/localmeta/kylin.properties| 2 +-
 examples/test_case_data/sandbox/kylin.properties  | 2 +-
 kylin-it/pom.xml  | 2 +-
 13 files changed, 33 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 2154c32..4a9804d 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -31,12 +31,17 @@ public abstract class AbstractKylinTestCase {
 public static final String[] SERVICES_WITH_CACHE = { //
 "org.apache.kylin.cube.CubeManager", //
 "org.apache.kylin.cube.CubeDescManager", //
+"org.apache.kylin.dict.lookup.SnapshotManager", //
+"org.apache.kylin.dict.DictionaryManager", //
 "org.apache.kylin.storage.hybrid.HybridManager", //
 "org.apache.kylin.metadata.realization.RealizationRegistry", //
 "org.apache.kylin.metadata.project.ProjectManager", //
 "org.apache.kylin.metadata.MetadataManager", //
+"org.apache.kylin.metadata.cachesync.Broadcaster", //
+"org.apache.kylin.metadata.badquery.BadQueryHistoryManager", //
 "org.apache.kylin.job.impl.threadpool.DistributedScheduler", //
-"org.apache.kylin.job.manager.ExecutableManager", //
+"org.apache.kylin.job.execution.ExecutableManager", //
+"org.apache.kylin.job.dao.ExecutableDao" //
 };
 
 public abstract void createTestMetadata() throws Exception;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 085158a..b45d017 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -66,6 +66,12 @@ public class SnapshotManager {
 return r;
 }
 
+public static void clearCache() {
+synchronized (SERVICE_CACHE) {
+SERVICE_CACHE.clear();
+}
+}
+
 // 

 
 private KylinConfig config;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java 
b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 5cae5ac..96505e6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -64,6 +64,12 @@ public class ExecutableDao {
 return r;
 }
 
+public static void clearCache() {
+

[2/7] kylin git commit: KYLIN-2409 change inmem cubing to single thread by default

[liyang]

KYLIN-2409 change inmem cubing to single thread by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21969753
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21969753
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21969753

Branch: refs/heads/master-hbase1.x
Commit: 21969753cc66efa22bc9fc933af46346e9846631
Parents: b49c9e3
Author: shaofengshi 
Authored: Thu Jan 19 11:52:17 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:37:42 2017 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 6 +-
 .../kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java| 2 +-
 .../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 9 +
 .../org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java | 2 ++
 4 files changed, 17 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d6774ff..74903d5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -300,6 +300,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
 }
 
+public int getCubeAlgorithmInMemConcurrentThreads() {
+return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-concurrent-threads", 
"1"));
+}
+
 public boolean isIgnoreCubeSignatureInconsistency() {
 return 
Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", 
"false"));
 }
@@ -744,7 +748,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getYarnStatusCheckIntervalSeconds() {
-return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"60"));
+return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"10"));
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 651203a..c7a4a05 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -46,7 +46,7 @@ abstract public class AbstractInMemCubeBuilder {
 final protected CubeDesc cubeDesc;
 final protected Map dictionaryMap;
 
-protected int taskThreadCount = 4;
+protected int taskThreadCount = 1;
 protected int reserveMemoryMB = 100;
 
 public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc 
flatDesc, Map dictionaryMap) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 55c04c6..9158aa3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -36,6 +36,7 @@ import java.util.Map.Entry;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.common.util.Pair;
@@ -178,11 +179,16 @@ public class GTAggregateScanner implements IGTScanner {
 final List dumps;
 final int keyLength;
 final boolean[] compareMask;
+boolean compareAll = true;
 final BufferedMeasureCodec measureCodec;
 
 final Comparator bytesComparator = new Comparator() {
 @Override
 public int compare(byte[] o1, byte[] o2) {
+ 

[1/7] kylin git commit: KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter into RoaringBitmapCounter [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master-hbase1.x aa4e2088f -> 5feb9e928 (forced update)


KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter 
into RoaringBitmapCounter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38c3e7bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38c3e7bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38c3e7bf

Branch: refs/heads/master-hbase1.x
Commit: 38c3e7bf691ecdfd0f8d42fcc97065a0596be018
Parents: 61833d9
Author: gaodayue 
Authored: Thu Jan 19 15:23:34 2017 +0800
Committer: gaodayue 
Committed: Thu Jan 19 15:43:14 2017 +0800

--
 .../gridtable/AggregationCacheMemSizeTest.java  |  11 +-
 .../metadata/measure/MeasureCodecTest.java  |   5 +-
 .../kylin/measure/bitmap/BitmapAggregator.java  |  25 +---
 .../kylin/measure/bitmap/BitmapCounter.java |  28 +++-
 .../measure/bitmap/BitmapCounterFactory.java|  30 
 .../BitmapIntersectDistinctCountAggFunc.java|  16 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |  17 ++-
 .../kylin/measure/bitmap/BitmapSerializer.java  |   9 +-
 .../measure/bitmap/ImmutableBitmapCounter.java  | 108 --
 .../measure/bitmap/MutableBitmapCounter.java|  60 
 .../measure/bitmap/RoaringBitmapCounter.java| 147 +++
 .../bitmap/RoaringBitmapCounterFactory.java |  47 ++
 .../measure/AggregatorMemEstimateTest.java  |   5 +-
 .../measure/bitmap/BitmapAggregatorTest.java|  56 ++-
 .../kylin/measure/bitmap/BitmapCounterTest.java |  32 +---
 .../measure/bitmap/BitmapSerializerTest.java|   5 +-
 16 files changed, 345 insertions(+), 256 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 63c7672..f749fb4 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -33,7 +33,9 @@ import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
 import org.apache.kylin.measure.basic.DoubleSumAggregator;
 import org.apache.kylin.measure.basic.LongSumAggregator;
 import org.apache.kylin.measure.bitmap.BitmapAggregator;
-import org.apache.kylin.measure.bitmap.MutableBitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounterFactory;
+import org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory;
 import org.apache.kylin.measure.hllc.HLLCAggregator;
 import org.apache.kylin.measure.hllc.HLLCounter;
 import org.github.jamm.MemoryMeter;
@@ -43,13 +45,14 @@ import com.google.common.base.Stopwatch;
 
 public class AggregationCacheMemSizeTest {
 private static final MemoryMeter meter = new MemoryMeter();
-private static final MutableBitmapCounter[] bitmaps = new 
MutableBitmapCounter[5];
+private static final BitmapCounterFactory bitmapFactory = 
RoaringBitmapCounterFactory.INSTANCE;
+private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
 private static final Random random = new Random();
 
 // consider bitmaps with variant cardinality
 static {
 for (int i = 0; i < bitmaps.length; i++) {
-bitmaps[i] = new MutableBitmapCounter();
+bitmaps[i] = bitmapFactory.newBitmap();
 }
 
 final int totalBits = 1_000_000;
@@ -116,7 +119,7 @@ public class AggregationCacheMemSizeTest {
 }
 
 private BitmapAggregator createBitmapAggr(boolean lowCardinality) {
-MutableBitmapCounter counter = new MutableBitmapCounter();
+BitmapCounter counter = bitmapFactory.newBitmap();
 counter.orWith(lowCardinality ? bitmaps[0] : bitmaps[3]);
 
 BitmapAggregator result = new BitmapAggregator();

http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 97c9751..7129a5e 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -25,7 +25,8 @@ import java.nio.ByteBuffer;
 
 

[6/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/5feb9e92/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClassOld2)) {
 desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch 

[1/2] kylin git commit: KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase [Forced Update!]

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2374 870cbf269 -> 24cae5c73 (forced update)


KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f88801a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f88801a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f88801a

Branch: refs/heads/KYLIN-2374
Commit: 0f88801a1e696eb58c58ec05e57f296a2ca6542e
Parents: 1e4ae54
Author: xiefan46 <958034...@qq.com>
Authored: Wed Jan 11 10:00:19 2017 +0800
Committer: shaofengshi 
Committed: Fri Jan 20 17:43:38 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   9 +-
 .../common/persistence/ResourceStoreTest.java   |   3 +-
 .../test_case_data/sandbox/kylin.properties |   1 +
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 117 +++
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 +
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 198 +++
 7 files changed, 411 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0f88801a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 74903d5..29ad5eb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -181,7 +181,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 if (!root.endsWith("/")) {
 root += "/";
 }
-
+
 // make sure path qualified
 if (!root.contains("://")) {
 if (!root.startsWith("/"))
@@ -189,7 +189,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 else
 root = "hdfs://" + root;
 }
-
+
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
@@ -201,6 +201,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
+//for hdfs resource store
+public String getHDFSMetadataUrl() {
+return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
+}
+
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f88801a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 4c31a15..ddaf481 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -110,9 +110,10 @@ public class ResourceStoreTest {
 }
 
 // list
-NavigableSet list;
+NavigableSet list = null;
 
 list = store.listResources(dir1);
+System.out.println(list);
 assertTrue(list.contains(path1));
 assertTrue(list.contains(path2) == false);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f88801a/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 06f8e4b..2d3cd79 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -41,6 +41,7 @@ kylin.source.hive.client=cli
 # The metadata store in hbase
 kylin.metadata.url=kylin_default_instance@hbase
 
+
 # The storage for final cube file in hbase
 kylin.storage.url=hbase
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f88801a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
new file mode 100644
index 000..ef04957
--- 

[2/2] kylin git commit: Add Zookeeper Lock

[shaofengshi]

Add Zookeeper Lock

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24cae5c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24cae5c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24cae5c7

Branch: refs/heads/KYLIN-2374
Commit: 24cae5c73889dcca4e18b2de4e12b68415ef2d62
Parents: 0f88801
Author: xiefan46 <958034...@qq.com>
Authored: Fri Jan 20 09:48:17 2017 +0800
Committer: shaofengshi 
Committed: Fri Jan 20 17:43:47 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  13 +-
 .../storage/hdfs/ITHDFSResourceStoreTest.java   |  65 +-
 .../kylin/storage/hdfs/ITLockManagerTest.java   | 206 +++
 .../kylin/storage/hbase/HBaseResourceStore.java |   1 +
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 
 .../kylin/storage/hdfs/HDFSResourceStore.java   |  90 ++--
 .../apache/kylin/storage/hdfs/LockManager.java  | 117 +++
 .../apache/kylin/storage/hdfs/ResourceLock.java |  51 +
 .../kylin/storage/hdfs/ZookeeperConfig.java |  27 +++
 10 files changed, 483 insertions(+), 173 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/24cae5c7/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 29ad5eb..0602e9d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,6 +193,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
+public String getRawHdfsWorkingDirectory() {
+String root = getRequired("kylin.env.hdfs-working-dir");
+if (!root.endsWith("/")) {
+root += "/";
+}
+return root;
+}
+
 // 

 // METADATA
 // 

@@ -201,11 +209,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
-//for hdfs resource store
-public String getHDFSMetadataUrl() {
-return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
-}
-
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);

http://git-wip-us.apache.org/repos/asf/kylin/blob/24cae5c7/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index ef04957..41bbcc0 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -17,10 +17,8 @@
 */
 
 package org.apache.kylin.storage.hdfs;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.ResourceStoreTest;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -29,11 +27,6 @@ import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.junit.Assert.assertEquals;
 
 /**
  * Created by xiefan on 17-1-10.
@@ -53,65 +46,13 @@ public class ITHDFSResourceStoreTest extends 
HBaseMetadataTestCase {
 this.cleanupTestMetadata();
 }
 
-@Ignore
-@Test
-public void testHDFSUrl() throws Exception {
-assertEquals("kylin_default_instance_hdfs@hdfs", 
kylinConfig.getHDFSMetadataUrl());
-System.out.println("hdfs working dir : " + 
kylinConfig.getHdfsWorkingDirectory());
-}
-
 
-@Ignore
 @Test
-public void testMultiThreadWriteHDFS() throws Exception{
-//System.out.println(kylinConfig.getHdfsWorkingDirectory());
-final Path testDir = new 

[2/2] kylin git commit: Add Zookeeper Lock

[shaofengshi]

Add Zookeeper Lock

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/870cbf26
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/870cbf26
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/870cbf26

Branch: refs/heads/KYLIN-2374
Commit: 870cbf2697f91b068b56064c302a67d09e45a12e
Parents: a594da7
Author: xiefan46 <958034...@qq.com>
Authored: Fri Jan 20 09:48:17 2017 +0800
Committer: shaofengshi 
Committed: Fri Jan 20 16:13:05 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   2 +-
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 117 -
 .../kylin/storage/hbase/HBaseResourceStore.java |   1 +
 .../org/apache/kylin/storage/hdfs/Config.java   |  29 +
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 --
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 ---
 .../kylin/storage/hdfs/HDFSResourceStore.java   |  78 ---
 .../apache/kylin/storage/hdfs/LockManager.java  | 117 +
 .../apache/kylin/storage/hdfs/ResourceLock.java |  53 
 .../storage/hdfs/ExampleClientThatLocks.java|  50 +++
 .../kylin/storage/hdfs/FakeLimitedResource.java |  41 ++
 .../storage/hdfs/HDFSResourceStoreTest.java | 118 +
 .../kylin/storage/hdfs/LockManagerTest.java |  69 ++
 .../kylin/storage/hdfs/TestingServer.java   |  42 ++
 .../org/apache/kylin/storage/hdfs/ZkDemo.java   | 129 +++
 15 files changed, 713 insertions(+), 219 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/870cbf26/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 29ad5eb..a4d6ca0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -203,7 +203,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 //for hdfs resource store
 public String getHDFSMetadataUrl() {
-return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
+return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs_meta@hdfs");
 }
 
 // for test only

http://git-wip-us.apache.org/repos/asf/kylin/blob/870cbf26/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
deleted file mode 100644
index ef04957..000
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hdfs;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.ResourceStoreTest;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Created by xiefan on 17-1-10.
- */
-public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
-
-KylinConfig kylinConfig;
-
-@Before
-public void setup() throws Exception {
-

[1/2] kylin git commit: KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2374 [created] 870cbf269


KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a594da75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a594da75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a594da75

Branch: refs/heads/KYLIN-2374
Commit: a594da751bfef41512b43084c05d7dff57524f40
Parents: 1e4ae54
Author: xiefan46 <958034...@qq.com>
Authored: Wed Jan 11 10:00:19 2017 +0800
Committer: shaofengshi 
Committed: Fri Jan 20 16:12:54 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   9 +-
 .../common/persistence/ResourceStoreTest.java   |   3 +-
 .../test_case_data/sandbox/kylin.properties |   1 +
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 117 +++
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 +
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 198 +++
 7 files changed, 411 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a594da75/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 74903d5..29ad5eb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -181,7 +181,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 if (!root.endsWith("/")) {
 root += "/";
 }
-
+
 // make sure path qualified
 if (!root.contains("://")) {
 if (!root.startsWith("/"))
@@ -189,7 +189,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 else
 root = "hdfs://" + root;
 }
-
+
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
@@ -201,6 +201,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
+//for hdfs resource store
+public String getHDFSMetadataUrl() {
+return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
+}
+
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);

http://git-wip-us.apache.org/repos/asf/kylin/blob/a594da75/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 4c31a15..ddaf481 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -110,9 +110,10 @@ public class ResourceStoreTest {
 }
 
 // list
-NavigableSet list;
+NavigableSet list = null;
 
 list = store.listResources(dir1);
+System.out.println(list);
 assertTrue(list.contains(path1));
 assertTrue(list.contains(path2) == false);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a594da75/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 06f8e4b..2d3cd79 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -41,6 +41,7 @@ kylin.source.hive.client=cli
 # The metadata store in hbase
 kylin.metadata.url=kylin_default_instance@hbase
 
+
 # The storage for final cube file in hbase
 kylin.storage.url=hbase
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a594da75/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
new file mode 100644
index 000..ef04957
--- /dev/null
+++ 

kylin git commit: Make integration test to building cube with spark

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/spark-it 36980c38b -> 387fa1458


Make integration test to building cube with spark


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/387fa145
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/387fa145
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/387fa145

Branch: refs/heads/spark-it
Commit: 387fa1458e950fb588aa38c006ba7fa514dd7f23
Parents: 36980c3
Author: shaofengshi 
Authored: Fri Jan 20 16:03:55 2017 +0800
Committer: shaofengshi 
Committed: Fri Jan 20 16:03:55 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 ++
 .../measure/bitmap/RoaringBitmapCounter.java|  3 ++-
 .../bitmap/RoaringBitmapCounterFactory.java |  3 ++-
 .../measure/percentile/PercentileCounter.java   | 22 +++-
 .../apache/kylin/engine/spark/SparkCubing.java  |  5 -
 kylin-it/pom.xml| 19 +
 6 files changed, 54 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 35f5f7a..1734bee 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -68,6 +68,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 return sparkHome;
 }
 
+sparkHome = System.getProperty("SPARK_HOME");
+if (StringUtils.isNotEmpty(sparkHome)) {
+logger.info("SPARK_HOME was set to " + sparkHome);
+return sparkHome;
+}
+
 return getKylinHome() + File.separator + "spark";
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index cd07d20..3d6d3e4 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -24,6 +24,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.Serializable;
 import java.nio.BufferOverflowException;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
@@ -31,7 +32,7 @@ import java.util.Iterator;
 /**
  * A {@link BitmapCounter} based on roaring bitmap.
  */
-public class RoaringBitmapCounter implements BitmapCounter {
+public class RoaringBitmapCounter implements BitmapCounter, Serializable {
 
 private ImmutableRoaringBitmap bitmap;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
index a71df95..822afa2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
@@ -21,9 +21,10 @@ package org.apache.kylin.measure.bitmap;
 import org.roaringbitmap.buffer.MutableRoaringBitmap;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 
-public class RoaringBitmapCounterFactory implements BitmapCounterFactory {
+public class RoaringBitmapCounterFactory implements BitmapCounterFactory, 
Serializable {
 public static final BitmapCounterFactory INSTANCE = new 
RoaringBitmapCounterFactory();
 
 private RoaringBitmapCounterFactory() {}

http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
index bf505cf..18d3bba 100644
---