[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2016-11-13 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 9a4a3bf08 -> a2366b0d6 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2adfdcf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2adfdcf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-

[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-13 Thread lidong 
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2366b0d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2366b0d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2366b0d

Branch: refs/heads/yang21-cdh5.7
Commit: a2366b0d68af60eed1a76af6e782f6bd02bbf70b
Parents: e2adfdc
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Mon Nov 14 14:10:50 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a2366b0d/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2366b0d/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2366b0d/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2366b0d/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread lidong 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2adfdcf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2adfdcf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2adfdcf

Branch: refs/heads/yang21-cdh5.7
Commit: e2adfdcfce8fac45b21235f28b46b6691851fdf8
Parents: 61335d9
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Nov 14 14:08:18 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2adfdcf/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2016-11-13 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x b28fe630b -> e2adfdcfc (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2adfdcf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2adfdcf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread lidong 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2adfdcf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2adfdcf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2adfdcf

Branch: refs/heads/yang21-hbase1.x
Commit: e2adfdcfce8fac45b21235f28b46b6691851fdf8
Parents: 61335d9
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Nov 14 14:08:18 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2adfdcf/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

kylin git commit: KYLIN-2181 remove integer as fixed_length in test_kylin_cube_with_slr_empty desc

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master b6d8e8849 -> 38dfbb3ef


KYLIN-2181 remove integer as fixed_length in test_kylin_cube_with_slr_empty desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38dfbb3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38dfbb3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38dfbb3e

Branch: refs/heads/master
Commit: 38dfbb3ef31ca0490647aacf0666dce0f119835b
Parents: b6d8e88
Author: Hongbin Ma 
Authored: Mon Nov 14 13:54:12 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 13:54:12 2016 +0800

--
 .../localmeta/cube_desc/test_kylin_cube_with_slr_desc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/38dfbb3e/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index 4064fcb..9c9924c 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -120,7 +120,7 @@
   "encoding" : "dict"
 }, {
   "column" : "leaf_categ_id",
-  "encoding" : "fixed_length:18"
+  "encoding" : "dict"
 }, {
   "column" : "meta_categ_name",
   "encoding" : "dict"

[1/4] kylin git commit: KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers in the rowkey [Forced Update!]

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 5f867e218 -> 9a4a3bf08 (forced update)


KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding 
for integers in the rowkey


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61335d9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61335d9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61335d9c

Branch: refs/heads/yang21-cdh5.7
Commit: 61335d9ce8ff193bd22e881855d0c68e48896ca9
Parents: c23205d
Author: Hongbin Ma 
Authored: Mon Nov 14 09:12:51 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:12:51 2016 +0800

--
 .../gtrecord/GTCubeStorageQueryBase.java| 22 ++--
 1 file changed, 20 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61335d9c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index b51af59..d648469 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -34,7 +34,9 @@ import org.apache.kylin.cube.RawQueryLastHacker;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -120,7 +122,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 //set whether to aggr at storage
 context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, 
groupsD, singleValuesD));
 // set limit push down
-enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, 
groupsD, filter, sqlDigest.aggregations, context);
+enableStorageLimitIfPossible(cubeInstance.getDescriptor(), cuboid, 
groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, 
context);
 context.setFinalPushDownLimit(cubeInstance);
 // set cautious threshold to prevent out of memory
 setThresholdIfNecessary(dimensionsD, metrics, context);
@@ -412,9 +414,25 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 }
 }
 
-private void enableStorageLimitIfPossible(Cuboid cuboid, 
Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
+private void enableStorageLimitIfPossible(CubeDesc cubeDesc, Cuboid 
cuboid, Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
 boolean possible = true;
 
+for (TblColRef col : cuboid.getColumns()) {
+RowKeyColDesc rowKeyDesc = cubeDesc.getRowkey().getColDesc(col);
+if (rowKeyDesc != null) {
+String encodingName = rowKeyDesc.getEncodingName();
+
+//for numbers, if it's fixed_length encoding, order is no 
longer preserved
+if (encodingName != null && 
encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && //
+(col.getType().isNumberFamily() || 
col.getType().isIntegerFamily())) {
+possible = false;
+logger.info("Storage limit push down is impossible because 
integer/number is encoded as fixedlength");
+}
+} else {
+logger.warn("RowKeyColDesc for {} does not exist", col);
+}
+}
+
 boolean goodFilter = filter == null || 
TupleFilter.isEvaluableRecursively(filter);
 if (!goodFilter) {
 possible = false;

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-13 Thread mahongbin 
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a4a3bf0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a4a3bf0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a4a3bf0

Branch: refs/heads/yang21-cdh5.7
Commit: 9a4a3bf083dbe1f2199c913ea0749dc914544c65
Parents: b28fe63
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:18:18 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a4a3bf0/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a4a3bf0/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a4a3bf0/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a4a3bf0/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b28fe630
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b28fe630
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b28fe630

Branch: refs/heads/yang21-cdh5.7
Commit: b28fe630bea9d695f0c970ed8024c5327fa95160
Parents: 61335d9
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:16:37 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/3] kylin git commit: KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers in the rowkey [Forced Update!]

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 130c26df9 -> b28fe630b (forced update)


KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding 
for integers in the rowkey


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61335d9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61335d9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61335d9c

Branch: refs/heads/yang21-hbase1.x
Commit: 61335d9ce8ff193bd22e881855d0c68e48896ca9
Parents: c23205d
Author: Hongbin Ma 
Authored: Mon Nov 14 09:12:51 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:12:51 2016 +0800

--
 .../gtrecord/GTCubeStorageQueryBase.java| 22 ++--
 1 file changed, 20 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61335d9c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index b51af59..d648469 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -34,7 +34,9 @@ import org.apache.kylin.cube.RawQueryLastHacker;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -120,7 +122,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 //set whether to aggr at storage
 context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, 
groupsD, singleValuesD));
 // set limit push down
-enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, 
groupsD, filter, sqlDigest.aggregations, context);
+enableStorageLimitIfPossible(cubeInstance.getDescriptor(), cuboid, 
groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, 
context);
 context.setFinalPushDownLimit(cubeInstance);
 // set cautious threshold to prevent out of memory
 setThresholdIfNecessary(dimensionsD, metrics, context);
@@ -412,9 +414,25 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 }
 }
 
-private void enableStorageLimitIfPossible(Cuboid cuboid, 
Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
+private void enableStorageLimitIfPossible(CubeDesc cubeDesc, Cuboid 
cuboid, Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
 boolean possible = true;
 
+for (TblColRef col : cuboid.getColumns()) {
+RowKeyColDesc rowKeyDesc = cubeDesc.getRowkey().getColDesc(col);
+if (rowKeyDesc != null) {
+String encodingName = rowKeyDesc.getEncodingName();
+
+//for numbers, if it's fixed_length encoding, order is no 
longer preserved
+if (encodingName != null && 
encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && //
+(col.getType().isNumberFamily() || 
col.getType().isIntegerFamily())) {
+possible = false;
+logger.info("Storage limit push down is impossible because 
integer/number is encoded as fixedlength");
+}
+} else {
+logger.warn("RowKeyColDesc for {} does not exist", col);
+}
+}
+
 boolean goodFilter = filter == null || 
TupleFilter.isEvaluableRecursively(filter);
 if (!goodFilter) {
 possible = false;

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b28fe630
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b28fe630
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b28fe630

Branch: refs/heads/yang21-hbase1.x
Commit: b28fe630bea9d695f0c970ed8024c5327fa95160
Parents: 61335d9
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:16:37 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b28fe630/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

kylin git commit: KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding for integers in the rowkey

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21 c23205d93 -> 61335d9ce


KYLIN-2179 should disable limit push down if there exists fixed_lenth encoding 
for integers in the rowkey


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61335d9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61335d9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61335d9c

Branch: refs/heads/yang21
Commit: 61335d9ce8ff193bd22e881855d0c68e48896ca9
Parents: c23205d
Author: Hongbin Ma 
Authored: Mon Nov 14 09:12:51 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Nov 14 09:12:51 2016 +0800

--
 .../gtrecord/GTCubeStorageQueryBase.java| 22 ++--
 1 file changed, 20 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61335d9c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index b51af59..d648469 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -34,7 +34,9 @@ import org.apache.kylin.cube.RawQueryLastHacker;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
+import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -120,7 +122,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 //set whether to aggr at storage
 context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, 
groupsD, singleValuesD));
 // set limit push down
-enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, 
groupsD, filter, sqlDigest.aggregations, context);
+enableStorageLimitIfPossible(cubeInstance.getDescriptor(), cuboid, 
groups, derivedPostAggregation, groupsD, filter, sqlDigest.aggregations, 
context);
 context.setFinalPushDownLimit(cubeInstance);
 // set cautious threshold to prevent out of memory
 setThresholdIfNecessary(dimensionsD, metrics, context);
@@ -412,9 +414,25 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 }
 }
 
-private void enableStorageLimitIfPossible(Cuboid cuboid, 
Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
+private void enableStorageLimitIfPossible(CubeDesc cubeDesc, Cuboid 
cuboid, Collection groups, Set derivedPostAggregation, 
Collection groupsD, TupleFilter filter, Collection 
functionDescs, StorageContext context) {
 boolean possible = true;
 
+for (TblColRef col : cuboid.getColumns()) {
+RowKeyColDesc rowKeyDesc = cubeDesc.getRowkey().getColDesc(col);
+if (rowKeyDesc != null) {
+String encodingName = rowKeyDesc.getEncodingName();
+
+//for numbers, if it's fixed_length encoding, order is no 
longer preserved
+if (encodingName != null && 
encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && //
+(col.getType().isNumberFamily() || 
col.getType().isIntegerFamily())) {
+possible = false;
+logger.info("Storage limit push down is impossible because 
integer/number is encoded as fixedlength");
+}
+} else {
+logger.warn("RowKeyColDesc for {} does not exist", col);
+}
+}
+
 boolean goodFilter = filter == null || 
TupleFilter.isEvaluableRecursively(filter);
 if (!goodFilter) {
 possible = false;

kylin git commit: minor, fix sample cube metadata file rename

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master f1042bf01 -> b6d8e8849


minor, fix sample cube metadata file rename

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6d8e884
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6d8e884
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6d8e884

Branch: refs/heads/master
Commit: b6d8e8849751b12706669069c83a4bedddfc9be0
Parents: f1042bf
Author: Roger Shi 
Authored: Sun Nov 13 22:45:44 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:56:50 2016 +0800

--
 build/bin/sample.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d8e884/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index cc731cf..c1a7bcc 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -52,8 +52,8 @@ fi
 mkdir -p ${KYLIN_HOME}/sample_cube/metadata
 cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
 
-sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
-sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

[1/4] kylin git commit: minor, fix sample cube metadata file rename [Forced Update!]

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 c15167ee4 -> 5f867e218 (forced update)


minor, fix sample cube metadata file rename

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c23205d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c23205d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c23205d9

Branch: refs/heads/yang21-cdh5.7
Commit: c23205d9394d3149db5771779b54bfec346e60c0
Parents: 29ac66b
Author: Roger Shi 
Authored: Sun Nov 13 22:45:44 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:47:49 2016 +0800

--
 build/bin/sample.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c23205d9/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index cc731cf..c1a7bcc 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -52,8 +52,8 @@ fi
 mkdir -p ${KYLIN_HOME}/sample_cube/metadata
 cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
 
-sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
-sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/130c26df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/130c26df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/130c26df

Branch: refs/heads/yang21-cdh5.7
Commit: 130c26df9da803afa5cace919c6afe673b01f2f5
Parents: c23205d
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:48:46 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-13 Thread mahongbin 
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f867e21
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f867e21
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f867e21

Branch: refs/heads/yang21-cdh5.7
Commit: 5f867e218e8d047f111b8b5e3ba6b601116ac361
Parents: 130c26d
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:50:23 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f867e21/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f867e21/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f867e21/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f867e21/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/130c26df/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/3] kylin git commit: minor, fix sample cube metadata file rename [Forced Update!]

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 1e0a6885c -> 130c26df9 (forced update)


minor, fix sample cube metadata file rename

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c23205d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c23205d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c23205d9

Branch: refs/heads/yang21-hbase1.x
Commit: c23205d9394d3149db5771779b54bfec346e60c0
Parents: 29ac66b
Author: Roger Shi 
Authored: Sun Nov 13 22:45:44 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:47:49 2016 +0800

--
 build/bin/sample.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c23205d9/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index cc731cf..c1a7bcc 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -52,8 +52,8 @@ fi
 mkdir -p ${KYLIN_HOME}/sample_cube/metadata
 cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
 
-sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
-sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

kylin git commit: minor, fix sample cube metadata file rename

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21 29ac66bb1 -> c23205d93


minor, fix sample cube metadata file rename

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c23205d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c23205d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c23205d9

Branch: refs/heads/yang21
Commit: c23205d9394d3149db5771779b54bfec346e60c0
Parents: 29ac66b
Author: Roger Shi 
Authored: Sun Nov 13 22:45:44 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 22:47:49 2016 +0800

--
 build/bin/sample.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c23205d9/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index cc731cf..c1a7bcc 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -52,8 +52,8 @@ fi
 mkdir -p ${KYLIN_HOME}/sample_cube/metadata
 cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
 
-sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
-sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[2/5] kylin git commit: minor, refine sample cube metadata

2016-11-13 Thread mahongbin 
minor, refine sample cube metadata

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29ac66bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29ac66bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29ac66bb

Branch: refs/heads/yang21-cdh5.7
Commit: 29ac66bb171bb2cab4aceff0de24bd985114fa82
Parents: 07bd53b
Author: Roger Shi 
Authored: Sun Nov 13 17:31:21 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 13 19:10:22 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../template/cube/kylin_sales_cube.json |   2 +-
 .../template/cube_desc/kylin_sales_cube.json| 199 +++
 .../cube_desc/kylin_sales_cube_desc.json| 199 ---
 4 files changed, 201 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 11d995c..973ed17 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -237,7 +237,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.error("Cannot found kafka dependency jars: " + e);
+logger.debug("Cannot found kafka dependency jars: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube/kylin_sales_cube.json 
b/examples/sample_cube/template/cube/kylin_sales_cube.json
index 86042ba..3d6d299 100644
--- a/examples/sample_cube/template/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube/kylin_sales_cube.json
@@ -4,7 +4,7 @@
   "last_modified" : 0,
   "name" : "kylin_sales_cube",
   "owner" : null,
-  "descriptor" : "kylin_sales_cube_desc",
+  "descriptor" : "kylin_sales_cube",
   "status" : "DISABLED",
   "segments" : [ ],
   "create_time" : null,

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
new file mode 100644
index 000..eb17922
--- /dev/null
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -0,0 +1,199 @@
+{
+  "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
+  "name" : "kylin_sales_cube",
+  "model_name" : "kylin_sales_model",
+  "description" : null,
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "CAL_DT",
+"table" : "DEFAULT.KYLIN_CAL_DT",
+"column" : "{FK}",
+"derived" : [ "WEEK_BEG_DT" ]
+  }, {
+"name" : "CATEGORY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "{FK}",
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "META_CATEG_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL2_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL3_NAME",
+"derived" : null
+  }, {
+"name" : "LSTG_FORMAT_NAME",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "LSTG_FORMAT_NAME",
+"derived" : null
+  }, {
+"name" : "USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "USER_ID",
+"derived" : null
+  }, {
+"name" : "REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "REGION",
+"derived" : null
+  } ],
+  "measures" : [ {
+"name" : "GMV_SUM",
+"function" : {
+  "expression" : "SUM",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : null
+  },
+  "returntype" : "decimal(19,4)"
+},
+"dependent_measure_ref" : null
+  }, {
+"name" : "GMV_MIN",
+"function" : {
+

[5/5] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-13 Thread mahongbin 
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c15167ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c15167ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c15167ee

Branch: refs/heads/yang21-cdh5.7
Commit: c15167ee49d30dea6750d89f2f9bd647c3bdbade
Parents: 1e0a688
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Sun Nov 13 21:40:20 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c15167ee/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/c15167ee/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/c15167ee/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c15167ee/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[1/5] kylin git commit: set ResourceStore verbose logs as Trace level [Forced Update!]

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 cde86bef3 -> c15167ee4 (forced update)


set ResourceStore verbose logs as Trace level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07bd53b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07bd53b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07bd53b8

Branch: refs/heads/yang21-cdh5.7
Commit: 07bd53b8e63565ccb0b720f7395a6e9127a24946
Parents: ac3f3b6
Author: Hongbin Ma 
Authored: Sun Nov 13 17:04:27 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 17:04:27 2016 +0800

--
 .../java/org/apache/kylin/common/persistence/ResourceStore.java  | 4 ++--
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07bd53b8/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 532e7aa..466d9d2 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -204,7 +204,7 @@ abstract public class ResourceStore {
  */
 final public void putResource(String resPath, InputStream content, long 
ts) throws IOException {
 resPath = norm(resPath);
-logger.debug("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 putResourceImpl(resPath, content, ts);
 }
 
@@ -255,7 +255,7 @@ abstract public class ResourceStore {
  * delete a resource, does nothing on a folder
  */
 final public void deleteResource(String resPath) throws IOException {
-logger.debug("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 deleteResourceImpl(norm(resPath));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/07bd53b8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index e2f3661..8deb811 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -259,7 +259,7 @@ public class HBaseResourceStore extends ResourceStore {
 Put put = buildPut(resPath, newTS, row, content, table);
 
 boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, 
put);
-logger.debug("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
+logger.trace("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
 if (!ok) {
 long real = getResourceTimestampImpl(resPath);
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e0a6885
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e0a6885
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e0a6885

Branch: refs/heads/yang21-cdh5.7
Commit: 1e0a6885ceb5399dbe614ab1930625b4d26a7a34
Parents: 29ac66b
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 21:38:43 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[1/4] kylin git commit: set ResourceStore verbose logs as Trace level [Forced Update!]

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x f6735a811 -> 1e0a6885c (forced update)


set ResourceStore verbose logs as Trace level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07bd53b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07bd53b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07bd53b8

Branch: refs/heads/yang21-hbase1.x
Commit: 07bd53b8e63565ccb0b720f7395a6e9127a24946
Parents: ac3f3b6
Author: Hongbin Ma 
Authored: Sun Nov 13 17:04:27 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 17:04:27 2016 +0800

--
 .../java/org/apache/kylin/common/persistence/ResourceStore.java  | 4 ++--
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07bd53b8/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 532e7aa..466d9d2 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -204,7 +204,7 @@ abstract public class ResourceStore {
  */
 final public void putResource(String resPath, InputStream content, long 
ts) throws IOException {
 resPath = norm(resPath);
-logger.debug("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 putResourceImpl(resPath, content, ts);
 }
 
@@ -255,7 +255,7 @@ abstract public class ResourceStore {
  * delete a resource, does nothing on a folder
  */
 final public void deleteResource(String resPath) throws IOException {
-logger.debug("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 deleteResourceImpl(norm(resPath));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/07bd53b8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index e2f3661..8deb811 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -259,7 +259,7 @@ public class HBaseResourceStore extends ResourceStore {
 Put put = buildPut(resPath, newTS, row, content, table);
 
 boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, 
put);
-logger.debug("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
+logger.trace("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
 if (!ok) {
 long real = getResourceTimestampImpl(resPath);
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);

[2/4] kylin git commit: minor, refine sample cube metadata

2016-11-13 Thread mahongbin 
minor, refine sample cube metadata

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29ac66bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29ac66bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29ac66bb

Branch: refs/heads/yang21-hbase1.x
Commit: 29ac66bb171bb2cab4aceff0de24bd985114fa82
Parents: 07bd53b
Author: Roger Shi 
Authored: Sun Nov 13 17:31:21 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 13 19:10:22 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../template/cube/kylin_sales_cube.json |   2 +-
 .../template/cube_desc/kylin_sales_cube.json| 199 +++
 .../cube_desc/kylin_sales_cube_desc.json| 199 ---
 4 files changed, 201 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 11d995c..973ed17 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -237,7 +237,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.error("Cannot found kafka dependency jars: " + e);
+logger.debug("Cannot found kafka dependency jars: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube/kylin_sales_cube.json 
b/examples/sample_cube/template/cube/kylin_sales_cube.json
index 86042ba..3d6d299 100644
--- a/examples/sample_cube/template/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube/kylin_sales_cube.json
@@ -4,7 +4,7 @@
   "last_modified" : 0,
   "name" : "kylin_sales_cube",
   "owner" : null,
-  "descriptor" : "kylin_sales_cube_desc",
+  "descriptor" : "kylin_sales_cube",
   "status" : "DISABLED",
   "segments" : [ ],
   "create_time" : null,

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
new file mode 100644
index 000..eb17922
--- /dev/null
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -0,0 +1,199 @@
+{
+  "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
+  "name" : "kylin_sales_cube",
+  "model_name" : "kylin_sales_model",
+  "description" : null,
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "CAL_DT",
+"table" : "DEFAULT.KYLIN_CAL_DT",
+"column" : "{FK}",
+"derived" : [ "WEEK_BEG_DT" ]
+  }, {
+"name" : "CATEGORY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "{FK}",
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "META_CATEG_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL2_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL3_NAME",
+"derived" : null
+  }, {
+"name" : "LSTG_FORMAT_NAME",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "LSTG_FORMAT_NAME",
+"derived" : null
+  }, {
+"name" : "USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "USER_ID",
+"derived" : null
+  }, {
+"name" : "REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "REGION",
+"derived" : null
+  } ],
+  "measures" : [ {
+"name" : "GMV_SUM",
+"function" : {
+  "expression" : "SUM",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : null
+  },
+  "returntype" : "decimal(19,4)"
+},
+"dependent_measure_ref" : null
+  }, {
+"name" : "GMV_MIN",
+"function" : {
+

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e0a6885
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e0a6885
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e0a6885

Branch: refs/heads/yang21-hbase1.x
Commit: 1e0a6885ceb5399dbe614ab1930625b4d26a7a34
Parents: 29ac66b
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 21:38:43 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-13 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e0a6885/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

kylin git commit: minor, refine sample cube metadata

2016-11-13 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master b7600f87c -> f1042bf01


minor, refine sample cube metadata

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f1042bf0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f1042bf0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f1042bf0

Branch: refs/heads/master
Commit: f1042bf012d292c98e0d0506c10dfa0f1b2f6b7d
Parents: b7600f8
Author: Roger Shi 
Authored: Sun Nov 13 17:31:21 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 13 19:14:03 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../template/cube/kylin_sales_cube.json |   2 +-
 .../template/cube_desc/kylin_sales_cube.json| 199 +++
 .../cube_desc/kylin_sales_cube_desc.json| 199 ---
 4 files changed, 201 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f1042bf0/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 1fd1631..69387f8 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -237,7 +237,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.warn("Not found kafka client jar from classpath, it is 
optional for normal build: " + e);
+logger.debug("Not found kafka client jar from classpath, it is 
optional for normal build: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f1042bf0/examples/sample_cube/template/cube/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube/kylin_sales_cube.json 
b/examples/sample_cube/template/cube/kylin_sales_cube.json
index 86042ba..3d6d299 100644
--- a/examples/sample_cube/template/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube/kylin_sales_cube.json
@@ -4,7 +4,7 @@
   "last_modified" : 0,
   "name" : "kylin_sales_cube",
   "owner" : null,
-  "descriptor" : "kylin_sales_cube_desc",
+  "descriptor" : "kylin_sales_cube",
   "status" : "DISABLED",
   "segments" : [ ],
   "create_time" : null,

http://git-wip-us.apache.org/repos/asf/kylin/blob/f1042bf0/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
new file mode 100644
index 000..eb17922
--- /dev/null
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -0,0 +1,199 @@
+{
+  "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
+  "name" : "kylin_sales_cube",
+  "model_name" : "kylin_sales_model",
+  "description" : null,
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "CAL_DT",
+"table" : "DEFAULT.KYLIN_CAL_DT",
+"column" : "{FK}",
+"derived" : [ "WEEK_BEG_DT" ]
+  }, {
+"name" : "CATEGORY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "{FK}",
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "META_CATEG_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL2_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL3_NAME",
+"derived" : null
+  }, {
+"name" : "LSTG_FORMAT_NAME",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "LSTG_FORMAT_NAME",
+"derived" : null
+  }, {
+"name" : "USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "USER_ID",
+"derived" : null
+  }, {
+"name" : "REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "REGION",
+"derived" : null
+  } ],
+  "measures" : [ {
+"name" : "GMV_SUM",
+"function" : {
+  "expression" : "SUM",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : null
+

kylin git commit: minor, refine sample cube metadata

2016-11-13 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang21 07bd53b8e -> 29ac66bb1


minor, refine sample cube metadata

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29ac66bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29ac66bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29ac66bb

Branch: refs/heads/yang21
Commit: 29ac66bb171bb2cab4aceff0de24bd985114fa82
Parents: 07bd53b
Author: Roger Shi 
Authored: Sun Nov 13 17:31:21 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 13 19:10:22 2016 +0800

--
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../template/cube/kylin_sales_cube.json |   2 +-
 .../template/cube_desc/kylin_sales_cube.json| 199 +++
 .../cube_desc/kylin_sales_cube_desc.json| 199 ---
 4 files changed, 201 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 11d995c..973ed17 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -237,7 +237,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.error("Cannot found kafka dependency jars: " + e);
+logger.debug("Cannot found kafka dependency jars: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube/kylin_sales_cube.json 
b/examples/sample_cube/template/cube/kylin_sales_cube.json
index 86042ba..3d6d299 100644
--- a/examples/sample_cube/template/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube/kylin_sales_cube.json
@@ -4,7 +4,7 @@
   "last_modified" : 0,
   "name" : "kylin_sales_cube",
   "owner" : null,
-  "descriptor" : "kylin_sales_cube_desc",
+  "descriptor" : "kylin_sales_cube",
   "status" : "DISABLED",
   "segments" : [ ],
   "create_time" : null,

http://git-wip-us.apache.org/repos/asf/kylin/blob/29ac66bb/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
new file mode 100644
index 000..eb17922
--- /dev/null
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -0,0 +1,199 @@
+{
+  "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
+  "name" : "kylin_sales_cube",
+  "model_name" : "kylin_sales_model",
+  "description" : null,
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "CAL_DT",
+"table" : "DEFAULT.KYLIN_CAL_DT",
+"column" : "{FK}",
+"derived" : [ "WEEK_BEG_DT" ]
+  }, {
+"name" : "CATEGORY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "{FK}",
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "META_CATEG_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL2_NAME",
+"derived" : null
+  }, {
+"name" : "CATEGORY_HIERARCHY",
+"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"column" : "CATEG_LVL3_NAME",
+"derived" : null
+  }, {
+"name" : "LSTG_FORMAT_NAME",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "LSTG_FORMAT_NAME",
+"derived" : null
+  }, {
+"name" : "USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "USER_ID",
+"derived" : null
+  }, {
+"name" : "REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "REGION",
+"derived" : null
+  } ],
+  "measures" : [ {
+"name" : "GMV_SUM",
+"function" : {
+  "expression" : "SUM",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : null
+  },
+  "returntype" : "decimal(19,4)"
+},
+

[1/2] kylin git commit: ignore 'defaultCatalog' in sql

2016-11-13 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 bdc1f0718 -> 9cdf8e7fe


ignore 'defaultCatalog' in sql


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d91d8475
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d91d8475
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d91d8475

Branch: refs/heads/v1.6.0-rc2
Commit: d91d847530373e8b054032160c5669bd0e1aa122
Parents: bdc1f07
Author: shaofengshi 
Authored: Fri Nov 11 16:12:34 2016 +0800
Committer: shaofengshi 
Committed: Sun Nov 13 17:09:11 2016 +0800

--
 .../java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java| 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d91d8475/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
index 8d8d971..d8c8845 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
@@ -27,6 +27,7 @@ public class KeywordDefaultDirtyHack implements 
IQueryTransformer {
 // KYLIN-2108, DEFAULT is hive default database, but a sql keyword 
too, needs quote
 sql = sql.replace("DEFAULT.", "\"DEFAULT\".");
 sql = sql.replace("default.", "\"default\".");
+sql = sql.replace("defaultCatalog.", "");
 
 return sql;
 }

[2/2] kylin git commit: minor, remove unnecessary check in BaseCuboidMapperBase

2016-11-13 Thread shaofengshi 
minor, remove unnecessary check in BaseCuboidMapperBase


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9cdf8e7f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9cdf8e7f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9cdf8e7f

Branch: refs/heads/v1.6.0-rc2
Commit: 9cdf8e7feb04c4d79a46e8682d161d902b3a9bfe
Parents: d91d847
Author: Yang Li 
Authored: Sun Nov 13 13:12:11 2016 +0800
Committer: shaofengshi 
Committed: Sun Nov 13 18:34:32 2016 +0800

--
 .../kylin/cube/model/CubeJoinedFlatTableEnrich.java  | 11 ---
 .../kylin/engine/mr/steps/BaseCuboidMapperBase.java  |  2 --
 2 files changed, 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9cdf8e7f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index 8af2297..979af76 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -20,7 +20,6 @@ package org.apache.kylin.cube.model;
 
 import java.util.List;
 
-import org.apache.kylin.common.util.BytesSplitter;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
@@ -81,16 +80,6 @@ public class CubeJoinedFlatTableEnrich implements 
IJoinedFlatTableDesc {
 }
 }
 
-// sanity check the input record (in bytes) matches what's expected
-public void sanityCheck(BytesSplitter bytesSplitter) {
-int columnCount = flatDesc.getAllColumns().size();
-if (columnCount != bytesSplitter.getBufferSize()) {
-throw new IllegalArgumentException("Expect " + columnCount + " 
columns, but see " + bytesSplitter.getBufferSize() + " -- " + bytesSplitter);
-}
-
-// TODO: check data types here
-}
-
 public CubeDesc getCubeDesc() {
 return cubeDesc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9cdf8e7f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 4f0d3fd..38595a0 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -195,8 +195,6 @@ public class BaseCuboidMapperBase extends 
KylinMapper

[kylin] Git Push Summary

2016-11-13 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 [created] bdc1f0718

kylin git commit: set ResourceStore verbose logs as Trace level

2016-11-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master a1847b90d -> b7600f87c


set ResourceStore verbose logs as Trace level


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b7600f87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b7600f87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b7600f87

Branch: refs/heads/master
Commit: b7600f87c061e4979e9fdde046a0e8606ffba331
Parents: a1847b9
Author: Hongbin Ma 
Authored: Sun Nov 13 17:02:21 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Nov 13 17:02:21 2016 +0800

--
 .../java/org/apache/kylin/common/persistence/ResourceStore.java  | 4 ++--
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b7600f87/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 532e7aa..466d9d2 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -204,7 +204,7 @@ abstract public class ResourceStore {
  */
 final public void putResource(String resPath, InputStream content, long 
ts) throws IOException {
 resPath = norm(resPath);
-logger.debug("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Directly saving resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 putResourceImpl(resPath, content, ts);
 }
 
@@ -255,7 +255,7 @@ abstract public class ResourceStore {
  * delete a resource, does nothing on a folder
  */
 final public void deleteResource(String resPath) throws IOException {
-logger.debug("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
+logger.trace("Deleting resource " + resPath + " (Store " + 
kylinConfig.getMetadataUrl() + ")");
 deleteResourceImpl(norm(resPath));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7600f87/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 3fd6426..82cc93c 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -259,7 +259,7 @@ public class HBaseResourceStore extends ResourceStore {
 Put put = buildPut(resPath, newTS, row, content, table);
 
 boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, 
put);
-logger.debug("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
+logger.trace("Update row " + resPath + " from oldTs: " + oldTS + 
", to newTs: " + newTS + ", operation result: " + ok);
 if (!ok) {
 long real = getResourceTimestampImpl(resPath);
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);