[kylin] Git Push Summary

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1-cdh5.7 [deleted] fe1b21a34

[1/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2-cdh5.7 [created] 57cc5b7a5


KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/57cc5b7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/57cc5b7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/57cc5b7a

Branch: refs/heads/v1.6.0-rc2-cdh5.7
Commit: 57cc5b7a517613b770c18d2e2997418f2e66e563
Parents: 26b96c7
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: shaofengshi 
Committed: Sun Nov 20 11:38:18 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/57cc5b7a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/57cc5b7a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/57cc5b7a/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-19 Thread shaofengshi 
http://git-wip-us.apache.org/repos/asf/kylin/blob/26b96c76/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/26b96c76/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-19 Thread shaofengshi 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/26b96c76
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/26b96c76
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/26b96c76

Branch: refs/heads/v1.6.0-rc2-cdh5.7
Commit: 26b96c766a9c9e6c8683ed308aee5bada25504e7
Parents: 50933a6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi 
Committed: Sun Nov 20 11:38:18 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java|  2 +-
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 48 files changed, 392 insertions(+), 448 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/26b96c76/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.

[kylin] Git Push Summary

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1-hbase1.x [deleted] 8b748cae3

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2-hbase1.x [created] 0a758520e


http://git-wip-us.apache.org/repos/asf/kylin/blob/0a758520/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0a758520/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-19 Thread shaofengshi 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a758520
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a758520
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a758520

Branch: refs/heads/v1.6.0-rc2-hbase1.x
Commit: 0a758520e5e076e7ab9abde3a4da45d0f7009e31
Parents: 50933a6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi 
Committed: Sun Nov 20 11:30:12 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java|  2 +-
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 48 files changed, 392 insertions(+), 448 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0a758520/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.

kylin git commit: KYLIN-2178 fix the UT with jdk1.8 [Forced Update!]

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 dcb0bc333 -> 50933a6c8 (forced update)


KYLIN-2178 fix the UT with jdk1.8


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50933a6c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50933a6c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50933a6c

Branch: refs/heads/v1.6.0-rc2
Commit: 50933a6c80e447033c304746eecd88e6516142fc
Parents: 79909ba
Author: shaofengshi 
Authored: Sat Nov 19 17:20:01 2016 +0800
Committer: shaofengshi 
Committed: Sun Nov 20 09:37:57 2016 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   |  9 +++
 .../validation/rule/AggregationGroupRule.java   | 11 +
 .../kylin/cube/AggregationGroupRuleTest.java| 25 ++--
 .../org/apache/kylin/cube/CubeDescTest.java |  1 +
 .../ut_cube_desc_combination_int_overflow.json  |  4 ++--
 5 files changed, 28 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/50933a6c/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4a5747b..77e6179 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -616,15 +616,16 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 }
 
 if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
-logger.warn("Aggregation group " + index + " mandatory 
dimensions overlap with hierarchy dimensions: " + 
CollectionUtils.intersection(mandatoryDims, hierarchyDims));
+
+logger.warn("Aggregation group " + index + " mandatory 
dimensions overlap with hierarchy dimensions: " + 
Sets.intersection(mandatoryDims, hierarchyDims));
 }
 if (CollectionUtils.containsAny(mandatoryDims, jointDims)) {
-logger.warn("Aggregation group " + index + " mandatory 
dimensions overlap with joint dimensions: " + 
CollectionUtils.intersection(mandatoryDims, jointDims));
+logger.warn("Aggregation group " + index + " mandatory 
dimensions overlap with joint dimensions: " + Sets.intersection(mandatoryDims, 
jointDims));
 }
 
 if (CollectionUtils.containsAny(hierarchyDims, jointDims)) {
 logger.error("Aggregation group " + index + " hierarchy 
dimensions overlap with joint dimensions");
-throw new IllegalStateException("Aggregation group " + index + 
" hierarchy dimensions overlap with joint dimensions: " + 
CollectionUtils.intersection(hierarchyDims, jointDims));
+throw new IllegalStateException("Aggregation group " + index + 
" hierarchy dimensions overlap with joint dimensions: " + 
Sets.intersection(hierarchyDims, jointDims));
 }
 
 if (hasSingle(hierarchyDimsList)) {
@@ -689,7 +690,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set overlap = new TreeSet<>();
 for (Set dims : dimsList) {
 if (CollectionUtils.containsAny(existing, dims)) {
-overlap.addAll(CollectionUtils.intersection(existing, dims));
+overlap.addAll(Sets.intersection(existing, dims));
 }
 existing.addAll(dims);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/50933a6c/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index 341efe7..55cb844 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -18,12 +18,12 @@
 
 package org.apache.kylin.cube.model.validation.rule;
 
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 
+import com.google.common.collect.Sets;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -148,8 +148,11 @@ public class AggregationGroupRule implements 
IValidatorRule {
 int overlapHierarchies = 0;

kylin git commit: KYLIN-2178 fix the UT with jdk1.8

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 8b37b81ba -> dcb0bc333


KYLIN-2178 fix the UT with jdk1.8


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dcb0bc33
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dcb0bc33
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dcb0bc33

Branch: refs/heads/v1.6.0-rc2
Commit: dcb0bc3332d2adde2e281af9bddde7374df57cfa
Parents: 8b37b81
Author: shaofengshi 
Authored: Sun Nov 20 09:37:30 2016 +0800
Committer: shaofengshi 
Committed: Sun Nov 20 09:37:30 2016 +0800

--
 .../main/java/org/apache/kylin/cube/model/CubeDesc.java  |  9 +
 .../cube/model/validation/rule/AggregationGroupRule.java | 11 +++
 2 files changed, 12 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb0bc33/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4a5747b..77e6179 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -616,15 +616,16 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 }
 
 if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
-logger.warn("Aggregation group " + index + " mandatory 
dimensions overlap with hierarchy dimensions: " + 
CollectionUtils.intersection(mandatoryDims, hierarchyDims));
+
+logger.warn("Aggregation group " + index + " mandatory 
dimensions overlap with hierarchy dimensions: " + 
Sets.intersection(mandatoryDims, hierarchyDims));
 }
 if (CollectionUtils.containsAny(mandatoryDims, jointDims)) {
-logger.warn("Aggregation group " + index + " mandatory 
dimensions overlap with joint dimensions: " + 
CollectionUtils.intersection(mandatoryDims, jointDims));
+logger.warn("Aggregation group " + index + " mandatory 
dimensions overlap with joint dimensions: " + Sets.intersection(mandatoryDims, 
jointDims));
 }
 
 if (CollectionUtils.containsAny(hierarchyDims, jointDims)) {
 logger.error("Aggregation group " + index + " hierarchy 
dimensions overlap with joint dimensions");
-throw new IllegalStateException("Aggregation group " + index + 
" hierarchy dimensions overlap with joint dimensions: " + 
CollectionUtils.intersection(hierarchyDims, jointDims));
+throw new IllegalStateException("Aggregation group " + index + 
" hierarchy dimensions overlap with joint dimensions: " + 
Sets.intersection(hierarchyDims, jointDims));
 }
 
 if (hasSingle(hierarchyDimsList)) {
@@ -689,7 +690,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set overlap = new TreeSet<>();
 for (Set dims : dimsList) {
 if (CollectionUtils.containsAny(existing, dims)) {
-overlap.addAll(CollectionUtils.intersection(existing, dims));
+overlap.addAll(Sets.intersection(existing, dims));
 }
 existing.addAll(dims);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb0bc33/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index 341efe7..55cb844 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -18,12 +18,12 @@
 
 package org.apache.kylin.cube.model.validation.rule;
 
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 
+import com.google.common.collect.Sets;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;
@@ -148,8 +148,11 @@ public class AggregationGroupRule implements 
IValidatorRule {
 int overlapHierarchies = 0;
 if (agg.getSelectRule().hierarchy_dims != null) {
 for (String[] oneHierarchy : 
agg.getSelectRule().hierarchy_dims) {
-

[kylin] Git Push Summary

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 [deleted] c8b5852ca

kylin git commit: KYLIN-1971 bug fix, model matching must match joining tables

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 634aeaa94 -> 1d4f57af7


KYLIN-1971 bug fix, model matching must match joining tables


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d4f57af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d4f57af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d4f57af

Branch: refs/heads/master
Commit: 1d4f57af7cf0ab89a1e1acd10bb4d1aaec1012cf
Parents: 634aeaa
Author: Yang Li 
Authored: Sat Nov 19 21:58:21 2016 +0800
Committer: Yang Li 
Committed: Sun Nov 20 07:48:48 2016 +0800

--
 .../kylin/dict/NumberDictionaryBuilder.java |  3 +
 .../apache/kylin/metadata/model/ColumnDesc.java |  6 ++
 .../kylin/metadata/model/DataModelDesc.java |  2 +
 .../apache/kylin/metadata/model/JoinDesc.java   | 63 --
 .../apache/kylin/query/relnode/OLAPJoinRel.java |  1 +
 .../kylin/query/relnode/OLAPTableScan.java  |  1 +
 .../v1/coprocessor/observer/RowTypeTest.java| 69 
 7 files changed, 56 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
index 0655ce1..68a05d4 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
@@ -21,8 +21,11 @@ package org.apache.kylin.dict;
 import org.apache.kylin.common.util.Bytes;
 
 /**
+ * Use NumberDictionaryForestBuilder instead.
+ * 
  * @author yangli9
  */
+@Deprecated
 public class NumberDictionaryBuilder extends TrieDictionaryBuilder {
 
 NumberDictionary.NumberBytesCodec codec = new 
NumberDictionary.NumberBytesCodec(NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT);

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index e0184b4..7d9133d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -191,6 +191,12 @@ public class ColumnDesc implements Serializable {
 } else if (!name.equals(other.name))
 return false;
 
+if (table == null) {
+if (other.table != null)
+return false;
+} else if (!table.equals(other.table))
+return false;
+
 if (datatype == null) {
 if (other.datatype != null)
 return false;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7c39a25..139fcbe 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -369,6 +369,8 @@ public class DataModelDesc extends RootPersistentEntity {
 fkCols[i] = col;
 }
 join.setForeignKeyColumns(fkCols);
+
+join.sortByFK();
 
 // Validate join in dimension
 if (pkCols.length != fkCols.length) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 5beda0a..1dd8725 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import

kylin git commit: minor, drop unused test

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 a75bad87f -> c8b5852ca


minor, drop unused test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8b5852c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8b5852c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8b5852c

Branch: refs/heads/KYLIN-1971
Commit: c8b5852ca0107f1251606cf2a5cb4a8c3b32bfe9
Parents: a75bad8
Author: Yang Li 
Authored: Sat Nov 19 22:27:00 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 19 22:27:00 2016 +0800

--
 .../v1/coprocessor/observer/RowTypeTest.java| 69 
 1 file changed, 69 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c8b5852c/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
deleted file mode 100644
index 364be9a..000
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- * 
- */
-public class RowTypeTest extends LocalFileMetadataTestCase {
-
-@Before
-public void setUp() throws Exception {
-this.createTestMetadata();
-}
-
-@After
-public void after() throws Exception {
-this.cleanupTestMetadata();
-}
-
-@Test
-public void testSerialize() {
-
-CubeInstance cube = 
CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready");
-CubeDesc cubeDesc = cube.getDescriptor();
-long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
-Cuboid cuboid = Cuboid.findById(cubeDesc, baseCuboidId);
-
-CoprocessorRowType rowType = 
CoprocessorRowType.fromCuboid(cube.getLatestReadySegment(), cuboid);
-byte[] bytes = CoprocessorRowType.serialize(rowType);
-CoprocessorRowType copy = CoprocessorRowType.deserialize(bytes);
-
-assertTrue(Arrays.equals(rowType.columnSizes, copy.columnSizes));
-for (int i = 0; i < rowType.columns.length; i++) {
-assertEquals(rowType.columns[i].getColumnDesc(), 
copy.columns[i].getColumnDesc());
-}
-}
-}

[04/12] kylin git commit: KYLIN-2198 Add a framework to allow major changes in DimensionEncoding

2016-11-19 Thread mahongbin 
KYLIN-2198 Add a framework to allow major changes in DimensionEncoding


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1acc419
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1acc419
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1acc419

Branch: refs/heads/yang21-cdh5.7
Commit: e1acc4192a982f897489f52d1bbc836a5c207da6
Parents: c0c0814
Author: Hongbin Ma 
Authored: Wed Nov 16 14:47:53 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:11:34 2016 +0800

--
 .../apache/kylin/common/util/JacksonBean.java   | 55 +++
 .../apache/kylin/common/util/JacksonTest.java   | 39 
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |  2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  | 13 ++-
 .../dimension/DimensionEncodingFactory.java | 97 +++-
 .../kylin/measure/topn/TopNMeasureType.java | 12 ++-
 .../kylin/rest/controller/CubeController.java   | 12 +--
 7 files changed, 198 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
new file mode 100644
index 000..42357f2
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
+public class JacksonBean {
+
+@JsonProperty("a")
+private String a;
+@JsonProperty("b")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+private int b;
+
+public String getA() {
+return a;
+}
+
+public void setA(String a) {
+this.a = a;
+}
+
+public int getB() {
+return b;
+}
+
+public void setB(int b) {
+this.b = b;
+}
+
+@Override
+public String toString() {
+return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
new file mode 100644
index 000..81be7eb
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+
+import org.junit.Test;
+

[06/12] kylin git commit: KYLIN-2201 fix combination check overflow error

2016-11-19 Thread mahongbin 
KYLIN-2201 fix combination check overflow error

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9bd6a2ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9bd6a2ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9bd6a2ab

Branch: refs/heads/yang21-cdh5.7
Commit: 9bd6a2ab6b5fd8f11a80198012a519bfe4edac08
Parents: 02e723a
Author: Roger Shi 
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   |   6 +-
 .../kylin/cube/AggregationGroupRuleTest.java|  11 +
 .../org/apache/kylin/cube/CubeDescTest.java |   7 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 +++
 4 files changed, 419 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..c914ebb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -580,7 +580,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 throw new IllegalStateException("Aggregation group " + index + 
" select rule field not set");
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(includeDims, agg.getIncludes());
 
@@ -598,7 +598,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set jointDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
 if (jointDimsList.size() > 0) {
-combination = combination * (1 << jointDimsList.size());
+combination = combination * (1L << jointDimsList.size());
 }
 
 if (!includeDims.containsAll(mandatoryDims) || 
!includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) 
{
@@ -612,7 +612,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (combination > config.getCubeAggrGroupMaxCombination()) {
 String msg = "Aggregation group " + index + " has too many 
combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 
'kylin.cube.aggrgroup.max.combination' to a bigger value.";

http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 6c3d544..7fea440 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -115,6 +116,16 @@ public class AggregationGroupRuleTest {
 assertEquals("Aggregation group 0 joint columns overlap with more than 
1 dim in same hierarchy", (vContext.getResults()[0].getMessage()));
 }
 
+@Test
+public void testCombinationIntOverflow() throws IOException {
+ValidateContext vContext = new ValidateContext();
+CubeDesc desc = JsonUtil.readValue(new 
FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
+
+IValidatorRule rule = getAggregationGroupRule();
+rule.validate(desc, vContext);
+assertEquals(1, vContext.getResults().length);
+}
+
 public AggregationGroupRule getAggregationGroupRule() {
 AggregationGroupRule rule = new AggregationGroupRule() {
 @Override

[11/12] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-19 Thread mahongbin 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a07a5292
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a07a5292
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a07a5292

Branch: refs/heads/yang21-cdh5.7
Commit: a07a5292b6352940cf0e5993890bc0b057ce3c3d
Parents: 61dd19f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Nov 19 22:17:14 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[05/12] kylin git commit: KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198

2016-11-19 Thread mahongbin 
KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac4e9ec5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac4e9ec5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac4e9ec5

Branch: refs/heads/yang21-cdh5.7
Commit: ac4e9ec563b6e14e387c5803d441c8dbcbc06f87
Parents: e1acc41
Author: Hongbin Ma 
Authored: Wed Nov 16 14:48:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:14:24 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |   4 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   9 +-
 .../apache/kylin/dimension/IntegerDimEncV2.java | 228 +++
 .../apache/kylin/dimension/IntDimEncTest.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  |  18 +-
 5 files changed, 248 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 87d37be..d572e56 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.IntegerDimEncV2;
 import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -99,7 +99,7 @@ public class DimEncodingPreserveOrderTest {
 @Test
 public void testVLongDimEncPreserveOrder() {
 for (int i = 1; i <= successValue.size(); i++) {
-IntegerDimEnc enc = new IntegerDimEnc(i);
+IntegerDimEncV2 enc = new IntegerDimEncV2(i);
 List encodedValues = Lists.newArrayList();
 for (long value : successValue.get(i - 1)) {
 encodedValues.add(encode(enc, value));

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index e55a0a8..983af9a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -31,8 +31,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * replacement for IntegerDimEnc, the diff is VLongDimEnc supports negative 
values
+ * replacement for IntDimEnc, the diff is IntegerDimEnc supports negative 
values
+ * for IntegerDimEnc(N), the supported range is (-2^(8*N-1),2^(8*N-1))
+ *
+ * -2^(8*N-1) is not supported because the slot is reserved for null values.
+ * -2^(8*N-1) will be encoded with warn, and its output will be null
  */
+@Deprecated//due to a fatal bug (KYLIN-2191)
 public class IntegerDimEnc extends DimensionEncoding {
 private static final long serialVersionUID = 1L;
 
@@ -127,7 +132,7 @@ public class IntegerDimEnc extends DimensionEncoding {
 
 //only take useful bytes
 integer = integer & MASK[fixedLen];
-boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
+boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0;
 if (!positive) {
 integer |= (~MASK[fixedLen]);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ac4e9ec5/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
new file mode 100644
index 000..1a54664
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under

[09/12] kylin git commit: minor, fix UT

2016-11-19 Thread mahongbin 
minor, fix UT

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61dd19f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61dd19f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61dd19f6

Branch: refs/heads/yang21-cdh5.7
Commit: 61dd19f6f64e2fdcb3b09de5f4b1c171d942ed1c
Parents: d87b7a5
Author: Roger Shi 
Authored: Thu Nov 17 17:08:17 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 17 17:54:45 2016 +0800

--
 .../common/util/LocalFileMetadataTestCase.java  |   1 +
 .../kylin/cube/AggregationGroupRuleTest.java|  37 +-
 .../org/apache/kylin/cube/CubeDescTest.java |  13 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |   7 +
 .../validation/rule/DictionaryRuleTest.java |   3 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 -
 ..._cube_desc_combination_int_overflow.json.bad | 398 +
 .../cube_desc/ut_large_dimension_number.json| 585 ---
 .../ut_large_dimension_number.json.bad  | 585 +++
 9 files changed, 1036 insertions(+), 991 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61dd19f6/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index c3e2538..d351041 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
 
 public static String LOCALMETA_TEST_DATA = 
"../examples/test_case_data/localmeta";
+public static String LOCALMETA_TEMP_DATA = "../examples/test_metadata/";
 
 @Override
 public void createTestMetadata() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/61dd19f6/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 7fea440..7903017 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -33,15 +33,31 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.validation.IValidatorRule;
 import org.apache.kylin.cube.model.validation.ValidateContext;
 import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
+import org.apache.kylin.metadata.MetadataManager;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
-public class AggregationGroupRuleTest {
+public class AggregationGroupRuleTest extends LocalFileMetadataTestCase{
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+MetadataManager.clearCache();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
 
 @Test
 public void testGoodDesc() throws IOException {
 AggregationGroupRule rule = getAggregationGroupRule();
 
-for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/").listFiles()) {
+for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + 
"/cube_desc/").listFiles()) {
+if (!f.getName().endsWith("json")) {
+continue;
+}
 CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), 
CubeDesc.class);
 ValidateContext vContext = new ValidateContext();
 rule.validate(desc, vContext);
@@ -59,7 +75,7 @@ public class AggregationGroupRuleTest {
 }
 };
 
-for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/").listFiles()) {
+for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + 
"/cube_desc/").listFiles()) {
 System.out.println(f.getName());
 CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), 
CubeDesc.class);
 ValidateContext vContext = new ValidateContext();
@@ -74,7 +90,7 @@ public class AggregationGroupRuleTest {
 public void testGoodDesc2() throws IOException {
 
 ValidateContext vContext = new

[02/12] kylin git commit: KYLIN-2191 fix integer encoding and refine UT

2016-11-19 Thread mahongbin 
KYLIN-2191 fix integer encoding and refine UT

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3e7eb75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3e7eb75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3e7eb75

Branch: refs/heads/yang21-cdh5.7
Commit: b3e7eb751005b79c59958230832921ed4763e019
Parents: 2e9a59d
Author: Roger Shi 
Authored: Tue Nov 15 20:59:15 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:09 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |  85 +++-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  | 128 +++
 3 files changed, 129 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b3e7eb75/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index ffd43e5..87d37be 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -20,22 +20,68 @@ package org.apache.kylin.gridtable;
 
 import static org.junit.Assert.assertTrue;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Ordering;
 
 public class DimEncodingPreserveOrderTest {
+private static List successValue;
+private static List failValue;
+
+@BeforeClass
+public static void initTestValue() {
+successValue = new ArrayList<>();
+successValue.add(new long[] { -127, 0, 127 });
+successValue.add(new long[] { -32767, -127, 0, 127, 32767 });
+successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 
8388607 });
+successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 
127, 32767, 8388607, 2147483647L });
+successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, 
-32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L });
+successValue.add(new long[] { -140737488355327L, -549755813887L, 
-2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 
549755813887L, 140737488355327L });
+successValue.add(new long[] { -36028797018963967L, -140737488355327L, 
-549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 
2147483647L, 549755813887L, 140737488355327L, 36028797018963967L });
+successValue.add(new long[] { //
+-9223372036854775807L, //
+-36028797018963967L, //
+-140737488355327L, //
+-549755813887L, //
+-2147483647L, //
+-8388607, //
+-32767, //
+-127, //
+0, //
+127, // (2 ^ 7) - 1
+32767, // (2 ^ 15)  - 1
+8388607, // (2 ^ 23) - 1
+2147483647L, // (2 ^ 31) - 1
+549755813887L, // (2 ^ 39) - 1
+140737488355327L, // (2 ^ 47) - 1
+36028797018963967L, // (2 ^ 55) - 1
+9223372036854775807L }); // (2 ^ 63) - 1
+
+failValue = new ArrayList<>();
+failValue.add(new long[] { -128, 128 });
+failValue.add(new long[] { -32768, 32768 });
+failValue.add(new long[] { -8388608, 8388608 });
+failValue.add(new long[] { -2147483648L, 2147483648L });
+failValue.add(new long[] { -549755813888L, 549755813888L });
+failValue.add(new long[] { -140737488355328L, 140737488355328L });
+failValue.add(new long[] { -36028797018963968L, 36028797018963968L });
+failValue.add(new long[] { -9223372036854775808L });
+}
+
 @Test
 public void testOneMoreByteVLongDimEncPreserveOrder() {
+// TODO: better test
 OneMoreByteVLongDimEnc enc = new OneMoreByteVLongDimEnc(2);
 List encodedValues = Lists.newArrayList();

[08/12] kylin git commit: KYLIN-2199 bug fix

2016-11-19 Thread mahongbin 
KYLIN-2199 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d87b7a51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d87b7a51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d87b7a51

Branch: refs/heads/yang21-cdh5.7
Commit: d87b7a51282ffb2e678af65c20b5a395a2347d49
Parents: 9bd6a2a
Author: Hongbin Ma 
Authored: Thu Nov 17 16:46:59 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 16:48:53 2016 +0800

--
 .../org/apache/kylin/dimension/DimensionEncodingFactory.java   | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d87b7a51/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 242e003..b617a19 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -73,6 +73,12 @@ public abstract class DimensionEncodingFactory {
 
 Map result = Maps.newHashMap();
 for (Pair p : factoryMap.keySet()) {
+if (result.containsKey(p.getFirst())) {
+if (result.get(p.getFirst()) > p.getSecond()) {
+continue;//skip small versions
+}
+}
+
 result.put(p.getFirst(), p.getSecond());
 }
 result.put(DictionaryDimEnc.ENCODING_NAME, 1);

[07/12] kylin git commit: KYLIN-2197 fix gg group buildMandatoryColumnMask

2016-11-19 Thread mahongbin 
KYLIN-2197 fix gg group buildMandatoryColumnMask

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/02e723ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/02e723ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/02e723ae

Branch: refs/heads/yang21-cdh5.7
Commit: 02e723ae0c40847517c8e8c195cb723a5d70d275
Parents: ac4e9ec
Author: Li Yang 
Authored: Wed Nov 16 13:33:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  13 +
 .../cube_desc/ut_large_dimension_number.json| 585 +++
 .../model_desc/ut_large_dimension_number.json   |  26 +
 .../localmeta/table/DEFAULT.WIDE_TABLE.json | 217 +++
 5 files changed, 842 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 5f780d0..c7924c7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -124,7 +124,7 @@ public class AggregationGroup {
 for (String dim : mandatory_dims) {
 TblColRef hColumn = colNameAbbr.get(dim);
 Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
-mandatoryColumnMask |= 1 << index;
+mandatoryColumnMask |= (1L << index);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index ecb1200..bdceb0f 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -283,6 +284,18 @@ public class CuboidSchedulerTest extends 
LocalFileMetadataTestCase {
 assertEquals(cuboidScheduler.getCuboidCount(), sum);
 }
 
+@Test
+public void testCuboid_onlyBaseCuboid() {
+CubeDesc cube = 
getCubeDescManager().getCubeDesc("ut_large_dimension_number");
+CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
+assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+
+List spanningChild = 
scheduler.getSpanningCuboid(baseCuboid.getId());
+assertTrue(spanningChild.size() > 0);
+}
+
 public CubeDescManager getCubeDescManager() {
 return CubeDescManager.getInstance(getTestConfig());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json 
b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
new file mode 100644
index 000..c0e8568
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
@@ -0,0 +1,585 @@
+{
+  "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+  "last_modified" : 1479207711845,
+  "name" : "ut_large_dimension_number",
+  "model_name" : "ut_large_dimension_number",
+  "description" : "",
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "DEFAULT.WIDE_TABLE.A",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "A",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.B",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "B",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.C",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "C",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.D",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "D",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.E",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" :

[01/12] kylin git commit: KYLIN-2193 parameterise org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD [Forced Update!]

2016-11-19 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 a2366b0d6 -> 2107dc55f (forced update)


KYLIN-2193 parameterise 
org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e9a59d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e9a59d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e9a59d5

Branch: refs/heads/yang21-cdh5.7
Commit: 2e9a59d57712b6429e2bed71a14206e213162366
Parents: 61335d9
Author: Hongbin Ma 
Authored: Tue Nov 15 21:06:21 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 15 21:22:41 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  |  5 -
 .../storage/translate/DerivedFilterTranslator.java | 13 +
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e703a07..a91e42b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -480,7 +480,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 public float getCubeVisitTimeoutTimes() {
 return 
Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
 }
-
+
+public int getDerivedInThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20"));
+}
 public int getBadQueryStackTraceDepth() {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 60c3a01..54b4c31 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.translate;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.kv.RowKeyColumnOrder;
@@ -36,17 +37,18 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 /**
  * @author yangli9
- * 
  */
 public class DerivedFilterTranslator {
-
-private static final int IN_THRESHOLD = 5;
+
+private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
 
@@ -83,7 +85,10 @@ public class DerivedFilterTranslator {
 
 TupleFilter translated;
 boolean loosened;
-if (satisfyingHostRecords.size() > IN_THRESHOLD) {
+if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
+);
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);
 loosened = true;
 } else {

[10/12] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-19 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[12/12] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-19 Thread mahongbin 
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2107dc55
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2107dc55
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2107dc55

Branch: refs/heads/yang21-cdh5.7
Commit: 2107dc55f03d011f6d8c399a11aae0646fb79a27
Parents: a07a529
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Sat Nov 19 22:19:16 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2107dc55/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/2107dc55/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/2107dc55/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2107dc55/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[03/12] kylin git commit: fix compile

2016-11-19 Thread mahongbin 
fix compile


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c0814f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c0814f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c0814f

Branch: refs/heads/yang21-cdh5.7
Commit: c0c0814f85509efe19c809b4f1fc22d2c63bf57a
Parents: b3e7eb7
Author: Hongbin Ma 
Authored: Wed Nov 16 10:17:28 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:28 2016 +0800

--
 .../apache/kylin/storage/translate/DerivedFilterTranslator.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c0814f/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 54b4c31..83fa32d 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -47,7 +47,7 @@ import com.google.common.collect.Sets;
  * @author yangli9
  */
 public class DerivedFilterTranslator {
-
+
 private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
@@ -86,7 +86,7 @@ public class DerivedFilterTranslator {
 TupleFilter translated;
 boolean loosened;
 if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
-logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}", //
 satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
 );
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);

[04/11] kylin git commit: KYLIN-2198 Add a framework to allow major changes in DimensionEncoding

2016-11-19 Thread mahongbin 
KYLIN-2198 Add a framework to allow major changes in DimensionEncoding


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e1acc419
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e1acc419
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e1acc419

Branch: refs/heads/yang21-hbase1.x
Commit: e1acc4192a982f897489f52d1bbc836a5c207da6
Parents: c0c0814
Author: Hongbin Ma 
Authored: Wed Nov 16 14:47:53 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:11:34 2016 +0800

--
 .../apache/kylin/common/util/JacksonBean.java   | 55 +++
 .../apache/kylin/common/util/JacksonTest.java   | 39 
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |  2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  | 13 ++-
 .../dimension/DimensionEncodingFactory.java | 97 +++-
 .../kylin/measure/topn/TopNMeasureType.java | 12 ++-
 .../kylin/rest/controller/CubeController.java   | 12 +--
 7 files changed, 198 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
new file mode 100644
index 000..42357f2
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
+public class JacksonBean {
+
+@JsonProperty("a")
+private String a;
+@JsonProperty("b")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+private int b;
+
+public String getA() {
+return a;
+}
+
+public void setA(String a) {
+this.a = a;
+}
+
+public int getB() {
+return b;
+}
+
+public void setB(int b) {
+this.b = b;
+}
+
+@Override
+public String toString() {
+return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/e1acc419/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
new file mode 100644
index 000..81be7eb
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+
+import org.junit.Test;
+

[02/11] kylin git commit: KYLIN-2191 fix integer encoding and refine UT

2016-11-19 Thread mahongbin 
KYLIN-2191 fix integer encoding and refine UT

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3e7eb75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3e7eb75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3e7eb75

Branch: refs/heads/yang21-hbase1.x
Commit: b3e7eb751005b79c59958230832921ed4763e019
Parents: 2e9a59d
Author: Roger Shi 
Authored: Tue Nov 15 20:59:15 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:09 2016 +0800

--
 .../gridtable/DimEncodingPreserveOrderTest.java |  85 +++-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   2 +-
 .../kylin/dimension/IntegerDimEncTest.java  | 128 +++
 3 files changed, 129 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b3e7eb75/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index ffd43e5..87d37be 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -20,22 +20,68 @@ package org.apache.kylin.gridtable;
 
 import static org.junit.Assert.assertTrue;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
 import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Ordering;
 
 public class DimEncodingPreserveOrderTest {
+private static List successValue;
+private static List failValue;
+
+@BeforeClass
+public static void initTestValue() {
+successValue = new ArrayList<>();
+successValue.add(new long[] { -127, 0, 127 });
+successValue.add(new long[] { -32767, -127, 0, 127, 32767 });
+successValue.add(new long[] { -8388607, -32767, -127, 0, 127, 32767, 
8388607 });
+successValue.add(new long[] { -2147483647L, -8388607, -32767, -127, 0, 
127, 32767, 8388607, 2147483647L });
+successValue.add(new long[] { -549755813887L, -2147483647L, -8388607, 
-32767, -127, 0, 127, 32767, 8388607, 2147483647L, 549755813887L });
+successValue.add(new long[] { -140737488355327L, -549755813887L, 
-2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 2147483647L, 
549755813887L, 140737488355327L });
+successValue.add(new long[] { -36028797018963967L, -140737488355327L, 
-549755813887L, -2147483647L, -8388607, -32767, -127, 0, 127, 32767, 8388607, 
2147483647L, 549755813887L, 140737488355327L, 36028797018963967L });
+successValue.add(new long[] { //
+-9223372036854775807L, //
+-36028797018963967L, //
+-140737488355327L, //
+-549755813887L, //
+-2147483647L, //
+-8388607, //
+-32767, //
+-127, //
+0, //
+127, // (2 ^ 7) - 1
+32767, // (2 ^ 15)  - 1
+8388607, // (2 ^ 23) - 1
+2147483647L, // (2 ^ 31) - 1
+549755813887L, // (2 ^ 39) - 1
+140737488355327L, // (2 ^ 47) - 1
+36028797018963967L, // (2 ^ 55) - 1
+9223372036854775807L }); // (2 ^ 63) - 1
+
+failValue = new ArrayList<>();
+failValue.add(new long[] { -128, 128 });
+failValue.add(new long[] { -32768, 32768 });
+failValue.add(new long[] { -8388608, 8388608 });
+failValue.add(new long[] { -2147483648L, 2147483648L });
+failValue.add(new long[] { -549755813888L, 549755813888L });
+failValue.add(new long[] { -140737488355328L, 140737488355328L });
+failValue.add(new long[] { -36028797018963968L, 36028797018963968L });
+failValue.add(new long[] { -9223372036854775808L });
+}
+
 @Test
 public void testOneMoreByteVLongDimEncPreserveOrder() {
+// TODO: better test
 OneMoreByteVLongDimEnc enc = new OneMoreByteVLongDimEnc(2);
 List encodedValues = Lists.newArrayList();

[09/11] kylin git commit: minor, fix UT

2016-11-19 Thread mahongbin 
minor, fix UT

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61dd19f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61dd19f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61dd19f6

Branch: refs/heads/yang21-hbase1.x
Commit: 61dd19f6f64e2fdcb3b09de5f4b1c171d942ed1c
Parents: d87b7a5
Author: Roger Shi 
Authored: Thu Nov 17 17:08:17 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 17 17:54:45 2016 +0800

--
 .../common/util/LocalFileMetadataTestCase.java  |   1 +
 .../kylin/cube/AggregationGroupRuleTest.java|  37 +-
 .../org/apache/kylin/cube/CubeDescTest.java |  13 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |   7 +
 .../validation/rule/DictionaryRuleTest.java |   3 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 -
 ..._cube_desc_combination_int_overflow.json.bad | 398 +
 .../cube_desc/ut_large_dimension_number.json| 585 ---
 .../ut_large_dimension_number.json.bad  | 585 +++
 9 files changed, 1036 insertions(+), 991 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61dd19f6/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index c3e2538..d351041 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
 public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
 
 public static String LOCALMETA_TEST_DATA = 
"../examples/test_case_data/localmeta";
+public static String LOCALMETA_TEMP_DATA = "../examples/test_metadata/";
 
 @Override
 public void createTestMetadata() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/61dd19f6/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 7fea440..7903017 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -33,15 +33,31 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.validation.IValidatorRule;
 import org.apache.kylin.cube.model.validation.ValidateContext;
 import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
+import org.apache.kylin.metadata.MetadataManager;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
-public class AggregationGroupRuleTest {
+public class AggregationGroupRuleTest extends LocalFileMetadataTestCase{
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+MetadataManager.clearCache();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
 
 @Test
 public void testGoodDesc() throws IOException {
 AggregationGroupRule rule = getAggregationGroupRule();
 
-for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/").listFiles()) {
+for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + 
"/cube_desc/").listFiles()) {
+if (!f.getName().endsWith("json")) {
+continue;
+}
 CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), 
CubeDesc.class);
 ValidateContext vContext = new ValidateContext();
 rule.validate(desc, vContext);
@@ -59,7 +75,7 @@ public class AggregationGroupRuleTest {
 }
 };
 
-for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/").listFiles()) {
+for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + 
"/cube_desc/").listFiles()) {
 System.out.println(f.getName());
 CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), 
CubeDesc.class);
 ValidateContext vContext = new ValidateContext();
@@ -74,7 +90,7 @@ public class AggregationGroupRuleTest {
 public void testGoodDesc2() throws IOException {
 
 ValidateContext vContext = new

[10/11] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-19 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[07/11] kylin git commit: KYLIN-2197 fix gg group buildMandatoryColumnMask

2016-11-19 Thread mahongbin 
KYLIN-2197 fix gg group buildMandatoryColumnMask

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/02e723ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/02e723ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/02e723ae

Branch: refs/heads/yang21-hbase1.x
Commit: 02e723ae0c40847517c8e8c195cb723a5d70d275
Parents: ac4e9ec
Author: Li Yang 
Authored: Wed Nov 16 13:33:08 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  13 +
 .../cube_desc/ut_large_dimension_number.json| 585 +++
 .../model_desc/ut_large_dimension_number.json   |  26 +
 .../localmeta/table/DEFAULT.WIDE_TABLE.json | 217 +++
 5 files changed, 842 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 5f780d0..c7924c7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -124,7 +124,7 @@ public class AggregationGroup {
 for (String dim : mandatory_dims) {
 TblColRef hColumn = colNameAbbr.get(dim);
 Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
-mandatoryColumnMask |= 1 << index;
+mandatoryColumnMask |= (1L << index);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index ecb1200..bdceb0f 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -283,6 +284,18 @@ public class CuboidSchedulerTest extends 
LocalFileMetadataTestCase {
 assertEquals(cuboidScheduler.getCuboidCount(), sum);
 }
 
+@Test
+public void testCuboid_onlyBaseCuboid() {
+CubeDesc cube = 
getCubeDescManager().getCubeDesc("ut_large_dimension_number");
+CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
+assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+
+List spanningChild = 
scheduler.getSpanningCuboid(baseCuboid.getId());
+assertTrue(spanningChild.size() > 0);
+}
+
 public CubeDescManager getCubeDescManager() {
 return CubeDescManager.getInstance(getTestConfig());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/02e723ae/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json 
b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
new file mode 100644
index 000..c0e8568
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
@@ -0,0 +1,585 @@
+{
+  "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+  "last_modified" : 1479207711845,
+  "name" : "ut_large_dimension_number",
+  "model_name" : "ut_large_dimension_number",
+  "description" : "",
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "DEFAULT.WIDE_TABLE.A",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "A",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.B",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "B",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.C",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "C",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.D",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "D",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.E",
+"table" : "DEFAULT.WIDE_TABLE",
+"column"

[01/11] kylin git commit: KYLIN-2193 parameterise org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD [Forced Update!]

2016-11-19 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x e2adfdcfc -> a07a5292b (forced update)


KYLIN-2193 parameterise 
org.apache.kylin.storage.translate.DerivedFilterTranslator#IN_THRESHOLD


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e9a59d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e9a59d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e9a59d5

Branch: refs/heads/yang21-hbase1.x
Commit: 2e9a59d57712b6429e2bed71a14206e213162366
Parents: 61335d9
Author: Hongbin Ma 
Authored: Tue Nov 15 21:06:21 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 15 21:22:41 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  |  5 -
 .../storage/translate/DerivedFilterTranslator.java | 13 +
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e703a07..a91e42b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -480,7 +480,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 public float getCubeVisitTimeoutTimes() {
 return 
Float.parseFloat(getOptional("kylin.query.cube.visit.timeout.times", "1"));
 }
-
+
+public int getDerivedInThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20"));
+}
 public int getBadQueryStackTraceDepth() {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e9a59d5/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 60c3a01..54b4c31 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.translate;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Array;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.kv.RowKeyColumnOrder;
@@ -36,17 +37,18 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 /**
  * @author yangli9
- * 
  */
 public class DerivedFilterTranslator {
-
-private static final int IN_THRESHOLD = 5;
+
+private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
 
@@ -83,7 +85,10 @@ public class DerivedFilterTranslator {
 
 TupleFilter translated;
 boolean loosened;
-if (satisfyingHostRecords.size() > IN_THRESHOLD) {
+if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
+);
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);
 loosened = true;
 } else {

[08/11] kylin git commit: KYLIN-2199 bug fix

2016-11-19 Thread mahongbin 
KYLIN-2199 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d87b7a51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d87b7a51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d87b7a51

Branch: refs/heads/yang21-hbase1.x
Commit: d87b7a51282ffb2e678af65c20b5a395a2347d49
Parents: 9bd6a2a
Author: Hongbin Ma 
Authored: Thu Nov 17 16:46:59 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 16:48:53 2016 +0800

--
 .../org/apache/kylin/dimension/DimensionEncodingFactory.java   | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d87b7a51/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 242e003..b617a19 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -73,6 +73,12 @@ public abstract class DimensionEncodingFactory {
 
 Map result = Maps.newHashMap();
 for (Pair p : factoryMap.keySet()) {
+if (result.containsKey(p.getFirst())) {
+if (result.get(p.getFirst()) > p.getSecond()) {
+continue;//skip small versions
+}
+}
+
 result.put(p.getFirst(), p.getSecond());
 }
 result.put(DictionaryDimEnc.ENCODING_NAME, 1);

[11/11] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-19 Thread mahongbin 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a07a5292
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a07a5292
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a07a5292

Branch: refs/heads/yang21-hbase1.x
Commit: a07a5292b6352940cf0e5993890bc0b057ce3c3d
Parents: 61dd19f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Nov 19 22:17:14 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a07a5292/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[03/11] kylin git commit: fix compile

2016-11-19 Thread mahongbin 
fix compile


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c0814f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c0814f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c0814f

Branch: refs/heads/yang21-hbase1.x
Commit: c0c0814f85509efe19c809b4f1fc22d2c63bf57a
Parents: b3e7eb7
Author: Hongbin Ma 
Authored: Wed Nov 16 10:17:28 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:05:28 2016 +0800

--
 .../apache/kylin/storage/translate/DerivedFilterTranslator.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c0814f/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 54b4c31..83fa32d 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -47,7 +47,7 @@ import com.google.common.collect.Sets;
  * @author yangli9
  */
 public class DerivedFilterTranslator {
-
+
 private static final Logger logger = 
LoggerFactory.getLogger(DerivedFilterTranslator.class);
 
 public static Pair translate(LookupStringTable 
lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {
@@ -86,7 +86,7 @@ public class DerivedFilterTranslator {
 TupleFilter translated;
 boolean loosened;
 if (satisfyingHostRecords.size() > 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
-logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}",//
+logger.info("Deciding to loosen filter on derived filter as host 
candidates number {} exceeds threshold {}", //
 satisfyingHostRecords.size(), 
KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
 );
 translated = buildRangeFilter(hostCols, satisfyingHostRecords);

[06/11] kylin git commit: KYLIN-2201 fix combination check overflow error

2016-11-19 Thread mahongbin 
KYLIN-2201 fix combination check overflow error

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9bd6a2ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9bd6a2ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9bd6a2ab

Branch: refs/heads/yang21-hbase1.x
Commit: 9bd6a2ab6b5fd8f11a80198012a519bfe4edac08
Parents: 02e723a
Author: Roger Shi 
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 17 11:50:09 2016 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   |   6 +-
 .../kylin/cube/AggregationGroupRuleTest.java|  11 +
 .../org/apache/kylin/cube/CubeDescTest.java |   7 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 +++
 4 files changed, 419 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..c914ebb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -580,7 +580,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 throw new IllegalStateException("Aggregation group " + index + 
" select rule field not set");
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(includeDims, agg.getIncludes());
 
@@ -598,7 +598,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set jointDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
 if (jointDimsList.size() > 0) {
-combination = combination * (1 << jointDimsList.size());
+combination = combination * (1L << jointDimsList.size());
 }
 
 if (!includeDims.containsAll(mandatoryDims) || 
!includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) 
{
@@ -612,7 +612,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (combination > config.getCubeAggrGroupMaxCombination()) {
 String msg = "Aggregation group " + index + " has too many 
combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 
'kylin.cube.aggrgroup.max.combination' to a bigger value.";

http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 6c3d544..7fea440 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -115,6 +116,16 @@ public class AggregationGroupRuleTest {
 assertEquals("Aggregation group 0 joint columns overlap with more than 
1 dim in same hierarchy", (vContext.getResults()[0].getMessage()));
 }
 
+@Test
+public void testCombinationIntOverflow() throws IOException {
+ValidateContext vContext = new ValidateContext();
+CubeDesc desc = JsonUtil.readValue(new 
FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
+
+IValidatorRule rule = getAggregationGroupRule();
+rule.validate(desc, vContext);
+assertEquals(1, vContext.getResults().length);
+}
+
 public AggregationGroupRule getAggregationGroupRule() {
 AggregationGroupRule rule = new AggregationGroupRule() {
 @Override

[kylin] Git Push Summary

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2006 [deleted] 248b181ce

[kylin] Git Push Summary

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2215 [deleted] a6e3ccd7b

kylin git commit: KYLIN-1971 bug fix, model matching must match joining tables

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 [created] a75bad87f


KYLIN-1971 bug fix, model matching must match joining tables


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a75bad87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a75bad87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a75bad87

Branch: refs/heads/KYLIN-1971
Commit: a75bad87fba2aad240dda226385eb2ca9396771e
Parents: 634aeaa
Author: Yang Li 
Authored: Sat Nov 19 21:58:21 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 19 21:58:21 2016 +0800

--
 .../kylin/dict/NumberDictionaryBuilder.java |  3 +
 .../apache/kylin/metadata/model/ColumnDesc.java |  6 ++
 .../kylin/metadata/model/DataModelDesc.java |  2 +
 .../apache/kylin/metadata/model/JoinDesc.java   | 63 +---
 .../apache/kylin/query/relnode/OLAPJoinRel.java |  1 +
 .../kylin/query/relnode/OLAPTableScan.java  |  1 +
 6 files changed, 56 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a75bad87/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
index 0655ce1..68a05d4 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
@@ -21,8 +21,11 @@ package org.apache.kylin.dict;
 import org.apache.kylin.common.util.Bytes;
 
 /**
+ * Use NumberDictionaryForestBuilder instead.
+ * 
  * @author yangli9
  */
+@Deprecated
 public class NumberDictionaryBuilder extends TrieDictionaryBuilder {
 
 NumberDictionary.NumberBytesCodec codec = new 
NumberDictionary.NumberBytesCodec(NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT);

http://git-wip-us.apache.org/repos/asf/kylin/blob/a75bad87/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index e0184b4..7d9133d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -191,6 +191,12 @@ public class ColumnDesc implements Serializable {
 } else if (!name.equals(other.name))
 return false;
 
+if (table == null) {
+if (other.table != null)
+return false;
+} else if (!table.equals(other.table))
+return false;
+
 if (datatype == null) {
 if (other.datatype != null)
 return false;

http://git-wip-us.apache.org/repos/asf/kylin/blob/a75bad87/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7c39a25..139fcbe 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -369,6 +369,8 @@ public class DataModelDesc extends RootPersistentEntity {
 fkCols[i] = col;
 }
 join.setForeignKeyColumns(fkCols);
+
+join.sortByFK();
 
 // Validate join in dimension
 if (pkCols.length != fkCols.length) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a75bad87/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 5beda0a..1dd8725 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
 
 /**
  */
@@ -98,6 +99,37 @@ public class

kylin git commit: KYLIN-1851 Treat empty dictionary as a normal case

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 2b2eb3405 -> 634aeaa94


KYLIN-1851 Treat empty dictionary as a normal case


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/634aeaa9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/634aeaa9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/634aeaa9

Branch: refs/heads/master
Commit: 634aeaa9486171bb816a738794f136c0d1936c92
Parents: 2b2eb34
Author: Yang Li 
Authored: Sat Nov 19 20:09:29 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 19 20:09:29 2016 +0800

--
 .../main/java/org/apache/kylin/dict/TrieDictionaryForest.java   | 5 -
 1 file changed, 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/634aeaa9/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 6525fe5..1015b11 100755
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -32,8 +32,6 @@ import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.common.util.Dictionary;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Use trie forest to optimize trie dictionary
@@ -57,8 +55,6 @@ public class TrieDictionaryForest extends Dictionary {
 
 private ArrayList maxValue;
 
-private static final Logger logger = 
LoggerFactory.getLogger(TrieDictionaryForest.class);
-
 public TrieDictionaryForest() { // default constructor for Writable 
interface
 }
 
@@ -377,7 +373,6 @@ public class TrieDictionaryForest extends Dictionary {
 private void initMaxValue() throws IllegalStateException {
 this.maxValue = new ArrayList<>();
 if (this.trees == null || trees.isEmpty()) {
-logger.info("Trees not init yet or trees size is zero. Could not 
init max value of each tree");
 return;
 }
 for (int i = 0; i < trees.size(); i++) {

kylin git commit: KYLIN-1851 fix empty dict bug

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master a6e3ccd7b -> 2b2eb3405


KYLIN-1851 fix empty dict bug

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b2eb340
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b2eb340
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b2eb340

Branch: refs/heads/master
Commit: 2b2eb3405fce57307496ba809e3da25c0bd9849d
Parents: a6e3ccd
Author: xiefan46 <958034...@qq.com>
Authored: Sat Nov 19 18:33:29 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 19 20:01:57 2016 +0800

--
 .../apache/kylin/dict/TrieDictionaryForest.java   | 12 +---
 .../kylin/dict/TrieDictionaryForestTest.java  | 18 ++
 2 files changed, 27 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b2eb340/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index eb33a5a..6525fe5 100755
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -32,6 +32,8 @@ import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.common.util.Dictionary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Use trie forest to optimize trie dictionary
@@ -55,11 +57,13 @@ public class TrieDictionaryForest extends Dictionary {
 
 private ArrayList maxValue;
 
+private static final Logger logger = 
LoggerFactory.getLogger(TrieDictionaryForest.class);
+
 public TrieDictionaryForest() { // default constructor for Writable 
interface
 }
 
 public TrieDictionaryForest(ArrayList trees, 
ArrayList valueDivide, //
-ArrayList accuOffset, 
BytesConverter bytesConverter, int baseId) {
+ArrayList accuOffset, BytesConverter bytesConverter, 
int baseId) {
 this.trees = trees;
 this.valueDivide = valueDivide;
 this.accuOffset = accuOffset;
@@ -372,8 +376,10 @@ public class TrieDictionaryForest extends Dictionary 
{
 
 private void initMaxValue() throws IllegalStateException {
 this.maxValue = new ArrayList<>();
-if (this.trees == null || trees.isEmpty())
-throw new IllegalStateException("Trees not init yet. Could not 
init max value of each tree");
+if (this.trees == null || trees.isEmpty()) {
+logger.info("Trees not init yet or trees size is zero. Could not 
init max value of each tree");
+return;
+}
 for (int i = 0; i < trees.size(); i++) {
 T curTreeMax = 
trees.get(i).getValueFromId(trees.get(i).getMaxId());
 byte[] b1 = bytesConvert.convertToBytes(curTreeMax);

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b2eb340/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
 
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index c4c0fd8..acd6894 100755
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -274,6 +274,24 @@ public class TrieDictionaryForestTest {
 }
 
 @Test
+public void emptyDictTest() throws Exception{
+TrieDictionaryForestBuilder b = new 
TrieDictionaryForestBuilder(new StringBytesConverter());
+TrieDictionaryForest dict = b.build();
+try{
+int id = dict.getIdFromValue("123",0);
+fail("id should not exist");
+}catch (IllegalArgumentException e){
+//right
+}
+try{
+String value = dict.getValueFromIdImpl(123);
+fail("value should not exist");
+}catch (IllegalArgumentException e){
+//right
+}
+}
+
+@Test
 public void roundingFlagTest() {
 ArrayList testData = new ArrayList<>();
 testData.add("b");

[kylin] Git Push Summary

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 [deleted] b5079074f

kylin git commit: KYLIN-2178 fix UT

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 79909baf5 -> 8b37b81ba


KYLIN-2178 fix UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8b37b81b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8b37b81b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8b37b81b

Branch: refs/heads/v1.6.0-rc2
Commit: 8b37b81ba8eceb50644daf25da2536d4062b1dd8
Parents: 79909ba
Author: shaofengshi 
Authored: Sat Nov 19 17:20:01 2016 +0800
Committer: shaofengshi 
Committed: Sat Nov 19 17:20:01 2016 +0800

--
 .../kylin/cube/AggregationGroupRuleTest.java| 25 ++--
 .../org/apache/kylin/cube/CubeDescTest.java |  1 +
 .../ut_cube_desc_combination_int_overflow.json  |  4 ++--
 3 files changed, 16 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8b37b81b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 563e139..4f42693 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileNotFoundException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -112,20 +112,21 @@ public class AggregationGroupRuleTest {
 IValidatorRule rule = getAggregationGroupRule();
 rule.validate(desc, vContext);
 vContext.print(System.out);
-assertEquals(1, vContext.getResults().length);
 assertEquals("Aggregation group 0 joint dimensions has overlap with 
more than 1 dimensions in same hierarchy: [CATEG_LVL2_NAME, META_CATEG_NAME]", 
(vContext.getResults()[0].getMessage()));
 }
 
-@Test
-public void testCombinationIntOverflow() throws IOException {
-ValidateContext vContext = new ValidateContext();
-CubeDesc desc = JsonUtil.readValue(new 
FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
-
-IValidatorRule rule = getAggregationGroupRule();
-rule.validate(desc, vContext);
-assertEquals(1, vContext.getResults().length);
-}
-
+@Test
+public void testCombinationIntOverflow() throws IOException {
+ValidateContext vContext = new ValidateContext();
+CubeDesc desc = JsonUtil.readValue(new 
FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + 
"/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
+
+desc.getAggregationGroups().get(0).getSelectRule().joint_dims = new 
String[][] { };
+
+IValidatorRule rule = getAggregationGroupRule();
+rule.validate(desc, vContext);
+assertEquals(1, vContext.getResults().length);
+}
+
 public AggregationGroupRule getAggregationGroupRule() {
 AggregationGroupRule rule = new AggregationGroupRule() {
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/8b37b81b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
--
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index ca7ae50..01e5090 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -196,6 +196,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase 
{
 public void testCombinationIntOverflow() throws  Exception {
 thrown.expect(IllegalStateException.class);
 CubeDesc cubeDesc = 
CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_cube_desc_combination_int_overflow");
+cubeDesc.getAggregationGroups().get(0).getSelectRule().joint_dims = 
new String[][] { };
 cubeDesc.init(getTestConfig());
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8b37b81b/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
 
b/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
index ec6f8f4..e8e2a52 100644
---

kylin git commit: KYLIN-2173 remove unused import

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 81a2fb752 -> 79909baf5


KYLIN-2173 remove unused import


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79909baf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79909baf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79909baf

Branch: refs/heads/v1.6.0-rc2
Commit: 79909baf599368bdca940b092ba591329c6748c5
Parents: 81a2fb7
Author: shaofengshi 
Authored: Sat Nov 19 16:37:46 2016 +0800
Committer: shaofengshi 
Committed: Sat Nov 19 16:37:46 2016 +0800

--
 .../org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/79909baf/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 434ff43..bd62f15 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -43,7 +43,6 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.PartitionDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.SQLDigest;

[1/3] kylin git commit: KYLIN-2197 fix gg group buildMandatoryColumnMask

2016-11-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 03305fa89 -> 81a2fb752


KYLIN-2197 fix gg group buildMandatoryColumnMask

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/791f570e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/791f570e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/791f570e

Branch: refs/heads/v1.6.0-rc2
Commit: 791f570ed877352953a5310833b0e317f20a31ef
Parents: 03305fa
Author: Li Yang 
Authored: Wed Nov 16 13:33:08 2016 +0800
Committer: shaofengshi 
Committed: Sat Nov 19 15:56:31 2016 +0800

--
 .../kylin/cube/model/AggregationGroup.java  |   2 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  |  13 +
 .../cube_desc/ut_large_dimension_number.json| 585 +++
 .../model_desc/ut_large_dimension_number.json   |  26 +
 .../localmeta/table/DEFAULT.WIDE_TABLE.json | 217 +++
 5 files changed, 842 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/791f570e/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 6e76ac3..3c88c62 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -140,7 +140,7 @@ public class AggregationGroup {
 for (String dim : mandatory_dims) {
 TblColRef hColumn = cubeDesc.getModel().findColumn(dim);
 Integer index = rowKeyDesc.getColumnBitIndex(hColumn);
-mandatoryColumnMask |= 1 << index;
+mandatoryColumnMask |= (1L << index);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/791f570e/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index ecb1200..bdceb0f 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeDescManager;
@@ -283,6 +284,18 @@ public class CuboidSchedulerTest extends 
LocalFileMetadataTestCase {
 assertEquals(cuboidScheduler.getCuboidCount(), sum);
 }
 
+@Test
+public void testCuboid_onlyBaseCuboid() {
+CubeDesc cube = 
getCubeDescManager().getCubeDesc("ut_large_dimension_number");
+CuboidScheduler scheduler = new CuboidScheduler(cube);
+
+Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
+assertTrue(Cuboid.isValid(cube, baseCuboid.getId()));
+
+List spanningChild = 
scheduler.getSpanningCuboid(baseCuboid.getId());
+assertTrue(spanningChild.size() > 0);
+}
+
 public CubeDescManager getCubeDescManager() {
 return CubeDescManager.getInstance(getTestConfig());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/791f570e/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json 
b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
new file mode 100644
index 000..c0e8568
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_large_dimension_number.json
@@ -0,0 +1,585 @@
+{
+  "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+  "last_modified" : 1479207711845,
+  "name" : "ut_large_dimension_number",
+  "model_name" : "ut_large_dimension_number",
+  "description" : "",
+  "null_string" : null,
+  "dimensions" : [ {
+"name" : "DEFAULT.WIDE_TABLE.A",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "A",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.B",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "B",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.C",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "C",
+"derived" : null
+  },{
+"name" : "DEFAULT.WIDE_TABLE.D",
+"table" : "DEFAULT.WIDE_TABLE",
+"column" : "D",
+"derived" :

[2/3] kylin git commit: KYLIN-2201 fix combination check overflow error

2016-11-19 Thread shaofengshi 
KYLIN-2201 fix combination check overflow error

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1bc8353
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1bc8353
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1bc8353

Branch: refs/heads/v1.6.0-rc2
Commit: a1bc83537b0a62af1f451db96c0c253c143458bd
Parents: 791f570
Author: Roger Shi 
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: shaofengshi 
Committed: Sat Nov 19 15:56:53 2016 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   |   6 +-
 .../validation/rule/AggregationGroupRule.java   |   4 +-
 .../kylin/cube/AggregationGroupRuleTest.java|  11 +
 .../org/apache/kylin/cube/CubeDescTest.java |   7 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 +++
 5 files changed, 421 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 47e9be8..6cbed80 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -568,7 +568,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 throw new IllegalStateException("Aggregation group " + index + 
" select rule field not set");
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(includeDims, agg.getIncludes());
 
@@ -586,7 +586,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 Set jointDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
 if (jointDimsList.size() > 0) {
-combination = combination * (1 << jointDimsList.size());
+combination = combination * (1L << jointDimsList.size());
 }
 
 if (!includeDims.containsAll(mandatoryDims) || 
!includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) 
{
@@ -607,7 +607,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (combination > config.getCubeAggrGroupMaxCombination()) {
 String msg = "Aggregation group " + index + " has too many 
combinations, use 'mandatory'/'hierarchy'/'joint' to optimize; or update 
'kylin.cube.aggrgroup.max.combination' to a bigger value.";

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index c1a062a..a5cf407 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -62,7 +62,7 @@ public class AggregationGroupRule implements 
IValidatorRule {
 continue;
 }
 
-int combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 if (agg.getIncludes() != null) {
 for (String include : agg.getIncludes()) {
@@ -115,7 +115,7 @@ public class AggregationGroupRule implements 
IValidatorRule {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1 << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
 Set intersection = new HashSet<>(mandatoryDims);

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1bc8353/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java

[3/3] kylin git commit: KYLIN-2178 fix CubeDescTest

2016-11-19 Thread shaofengshi 
KYLIN-2178 fix CubeDescTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81a2fb75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81a2fb75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81a2fb75

Branch: refs/heads/v1.6.0-rc2
Commit: 81a2fb7529a43d6d9e5ce5d36f0696aac4e64613
Parents: a1bc835
Author: shaofengshi 
Authored: Sat Nov 19 16:32:34 2016 +0800
Committer: shaofengshi 
Committed: Sat Nov 19 16:32:34 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/model/CubeDesc.java| 5 ++---
 .../kylin/cube/model/validation/rule/AggregationGroupRule.java | 6 +++---
 2 files changed, 5 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/81a2fb75/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 6cbed80..4a5747b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -29,7 +29,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -686,8 +685,8 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 }
 
 private Pair hasOverlap(ArrayList 
dimsList, Set Dims) {
-Set existing = new HashSet<>();
-Set overlap = new HashSet<>();
+Set existing = new TreeSet<>();
+Set overlap = new TreeSet<>();
 for (Set dims : dimsList) {
 if (CollectionUtils.containsAny(existing, dims)) {
 overlap.addAll(CollectionUtils.intersection(existing, dims));

http://git-wip-us.apache.org/repos/asf/kylin/blob/81a2fb75/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index a5cf407..341efe7 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -62,7 +62,7 @@ public class AggregationGroupRule implements 
IValidatorRule {
 continue;
 }
 
-long combination = 1;
+long combination = 1;
 Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 if (agg.getIncludes() != null) {
 for (String include : agg.getIncludes()) {
@@ -115,10 +115,10 @@ public class AggregationGroupRule implements 
IValidatorRule {
 normalDims.removeAll(hierarchyDims);
 normalDims.removeAll(jointDims);
 
-combination = combination * (1L << normalDims.size());
+combination = combination * (1L << normalDims.size());
 
 if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
-Set intersection = new HashSet<>(mandatoryDims);
+Set intersection = new TreeSet<>(mandatoryDims);
 intersection.retainAll(hierarchyDims);
 context.addResult(ResultLevel.ERROR, "Aggregation group " + 
index + " mandatory dimension has overlap with hierarchy dimension: " + 
intersection.toString());
 continue;

[2/2] kylin git commit: KYLIN-2215 Refactor DimensionEncoding.encode(byte[]) to encode(String)

2016-11-19 Thread liyang 
KYLIN-2215 Refactor DimensionEncoding.encode(byte[]) to encode(String)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a6e3ccd7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a6e3ccd7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a6e3ccd7

Branch: refs/heads/master
Commit: a6e3ccd7b22b19e03b5e3af568148816fe4a563a
Parents: d4a18c1
Author: Yang Li 
Authored: Sat Nov 19 16:07:10 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 19 16:07:10 2016 +0800

--
 .../kylin/cube/gridtable/TrimmedDimEnc.java |  2 +-
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|  2 +-
 .../apache/kylin/cube/kv/FuzzyMaskEncoder.java  |  4 +-
 .../org/apache/kylin/cube/kv/RowConstants.java  |  3 -
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|  4 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 33 +++--
 .../apache/kylin/cube/kv/RowKeyDecoderTest.java | 19 +++--
 .../apache/kylin/cube/kv/RowKeyEncoderTest.java | 42 +--
 .../gridtable/DimEncodingPreserveOrderTest.java | 18 +
 .../kylin/dimension/AbstractDateDimEnc.java | 17 +
 .../apache/kylin/dimension/BooleanDimEnc.java   | 19 ++---
 .../kylin/dimension/DictionaryDimEnc.java   |  7 +-
 .../kylin/dimension/DimensionEncoding.java  |  4 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  | 10 ++-
 .../kylin/dimension/FixedLenHexDimEnc.java  | 12 +--
 .../org/apache/kylin/dimension/IntDimEnc.java   | 12 +--
 .../apache/kylin/dimension/IntegerDimEnc.java   | 12 +--
 .../apache/kylin/dimension/IntegerDimEncV2.java | 12 +--
 .../kylin/dimension/OneMoreByteVLongDimEnc.java | 12 +--
 .../kylin/measure/topn/TopNMeasureType.java |  7 +-
 .../kylin/dimension/BooleanDimEncTest.java  |  6 +-
 .../kylin/dimension/FixedLenHexDimEncTest.java  |  6 +-
 .../apache/kylin/dimension/IntDimEncTest.java   |  6 +-
 .../kylin/dimension/IntegerDimEncTest.java  |  6 +-
 .../dimension/OneMoreByteVLongDimEncTest.java   |  6 +-
 .../engine/mr/steps/BaseCuboidMapperBase.java   | 77 +++-
 .../engine/mr/steps/HiveToBaseCuboidMapper.java |  9 +--
 .../coprocessor/CoprocessorProjector.java   |  4 +-
 .../common/coprocessor/FilterDecorator.java |  4 +-
 29 files changed, 122 insertions(+), 253 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e3ccd7/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
index c4bee8a..6d05668 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
@@ -44,7 +44,7 @@ public class TrimmedDimEnc extends DimensionEncoding {
 }
 
 @Override
-public void encode(byte[] value, int valueLen, byte[] output, int 
outputOffset) {
+public void encode(String valueStr, byte[] output, int outputOffset) {
 throw new UnsupportedOperationException();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e3ccd7/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
index 37b33aa..bfe6eb4 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
@@ -84,5 +84,5 @@ public abstract class AbstractRowKeyEncoder {
 
 abstract public byte[] encode(Map valueMap);
 
-abstract public byte[] encode(byte[][] values);
+abstract public byte[] encode(String[] values);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e3ccd7/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
index 94db94b..0cbb7d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
@@ -77,8 +77,8 @@ public class FuzzyMaskEncoder extends RowKeyEncoder {
 }
 
 @Override
-protected void fillColumnValue(TblColRef column, int columnLen, byte[] 
value, int valueLen, byte[] outputValue, int outputValueOffset) {
-if (value ==

[1/2] kylin git commit: minor, various code review

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master c8d4136b0 -> a6e3ccd7b


minor, various code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4a18c11
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4a18c11
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4a18c11

Branch: refs/heads/master
Commit: d4a18c113131dc316d78ad98e1eaf1fb18af95c7
Parents: c8d4136
Author: Yang Li 
Authored: Sat Nov 19 15:04:10 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 19 15:04:10 2016 +0800

--
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 14 +++
 .../storage/gtrecord/CubeScanRangePlanner.java  |  9 +++--
 .../mr/steps/NumberDictionaryForestTest.java| 39 
 .../kylin/provision/BuildCubeWithEngine.java|  5 +--
 4 files changed, 52 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d4a18c11/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index ff37752..672e1bc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -42,8 +42,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 private RowKeyColumnIO colIO;
 
 protected boolean enableSharding;
-private int UHCOffset = -1;//it's a offset to the beginning of body
-private int UHCLength = -1;
+private int uhcOffset = -1;//it's a offset to the beginning of body
+private int uhcLength = -1;
 
 public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) {
 super(cubeSeg, cuboid);
@@ -55,8 +55,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 colIO = new RowKeyColumnIO(cubeSeg.getDimensionEncodingMap());
 for (TblColRef column : cuboid.getColumns()) {
 if (shardByColumns.contains(column)) {
-UHCOffset = bodyLength;
-UHCLength = colIO.getColumnLength(column);
+uhcOffset = bodyLength;
+uhcLength = colIO.getColumnLength(column);
 }
 bodyLength += colIO.getColumnLength(column);
 }
@@ -72,8 +72,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 
 protected short calculateShard(byte[] key) {
 if (enableSharding) {
-int shardSeedOffset = UHCOffset == -1 ? 0 : UHCOffset;
-int shardSeedLength = UHCLength == -1 ? bodyLength : UHCLength;
+int shardSeedOffset = uhcOffset == -1 ? 0 : uhcOffset;
+int shardSeedLength = uhcLength == -1 ? bodyLength : uhcLength;
 short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());
 short shardOffset = ShardingHash.getShard(key, 
RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN + shardSeedOffset, shardSeedLength, 
cuboidShardNum);
 return 
ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), shardOffset, 
cubeSeg.getTotalShards(cuboid.getId()));
@@ -102,7 +102,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 }
 
 //ByteArray representing dimension does not have extra header
-public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, 
ByteArray buf, byte defaultValue) {
+private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, 
ByteArray buf, byte defaultValue) {
 int pos = 0;
 for (int i = 0; i < selectedCols.trueBitCount(); i++) {
 int c = selectedCols.trueBitAt(i);

http://git-wip-us.apache.org/repos/asf/kylin/blob/d4a18c11/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
index b011f40..8d5a3d4 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -71,7 +71,7 @@ public class CubeScanRangePlanner extends 
ScanRangePlannerBase {
 
 protected StorageContext context;
 
-public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, 
TupleFilter filter, Set dimensions, Set groupbyDims, //
+public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, 
TupleFilter filter, Set dimensions, Set groupByDims, //
 Collection metrics,

[kylin] Git Push Summary

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1971 [deleted] 44efded28

[1/2] kylin git commit: minor, various code review

2016-11-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2215 [created] a6e3ccd7b


minor, various code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4a18c11
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4a18c11
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4a18c11

Branch: refs/heads/KYLIN-2215
Commit: d4a18c113131dc316d78ad98e1eaf1fb18af95c7
Parents: c8d4136
Author: Yang Li 
Authored: Sat Nov 19 15:04:10 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 19 15:04:10 2016 +0800

--
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 14 +++
 .../storage/gtrecord/CubeScanRangePlanner.java  |  9 +++--
 .../mr/steps/NumberDictionaryForestTest.java| 39 
 .../kylin/provision/BuildCubeWithEngine.java|  5 +--
 4 files changed, 52 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d4a18c11/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index ff37752..672e1bc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -42,8 +42,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 private RowKeyColumnIO colIO;
 
 protected boolean enableSharding;
-private int UHCOffset = -1;//it's a offset to the beginning of body
-private int UHCLength = -1;
+private int uhcOffset = -1;//it's a offset to the beginning of body
+private int uhcLength = -1;
 
 public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) {
 super(cubeSeg, cuboid);
@@ -55,8 +55,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 colIO = new RowKeyColumnIO(cubeSeg.getDimensionEncodingMap());
 for (TblColRef column : cuboid.getColumns()) {
 if (shardByColumns.contains(column)) {
-UHCOffset = bodyLength;
-UHCLength = colIO.getColumnLength(column);
+uhcOffset = bodyLength;
+uhcLength = colIO.getColumnLength(column);
 }
 bodyLength += colIO.getColumnLength(column);
 }
@@ -72,8 +72,8 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 
 protected short calculateShard(byte[] key) {
 if (enableSharding) {
-int shardSeedOffset = UHCOffset == -1 ? 0 : UHCOffset;
-int shardSeedLength = UHCLength == -1 ? bodyLength : UHCLength;
+int shardSeedOffset = uhcOffset == -1 ? 0 : uhcOffset;
+int shardSeedLength = uhcLength == -1 ? bodyLength : uhcLength;
 short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());
 short shardOffset = ShardingHash.getShard(key, 
RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN + shardSeedOffset, shardSeedLength, 
cuboidShardNum);
 return 
ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), shardOffset, 
cubeSeg.getTotalShards(cuboid.getId()));
@@ -102,7 +102,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 }
 
 //ByteArray representing dimension does not have extra header
-public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, 
ByteArray buf, byte defaultValue) {
+private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, 
ByteArray buf, byte defaultValue) {
 int pos = 0;
 for (int i = 0; i < selectedCols.trueBitCount(); i++) {
 int c = selectedCols.trueBitAt(i);

http://git-wip-us.apache.org/repos/asf/kylin/blob/d4a18c11/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
index b011f40..8d5a3d4 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -71,7 +71,7 @@ public class CubeScanRangePlanner extends 
ScanRangePlannerBase {
 
 protected StorageContext context;
 
-public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, 
TupleFilter filter, Set dimensions, Set groupbyDims, //
+public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, 
TupleFilter filter, Set dimensions, Set groupByDims, //
 Collection

[2/2] kylin git commit: KYLIN-2215 Refactor DimensionEncoding.encode(byte[]) to encode(String)

2016-11-19 Thread liyang 
KYLIN-2215 Refactor DimensionEncoding.encode(byte[]) to encode(String)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a6e3ccd7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a6e3ccd7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a6e3ccd7

Branch: refs/heads/KYLIN-2215
Commit: a6e3ccd7b22b19e03b5e3af568148816fe4a563a
Parents: d4a18c1
Author: Yang Li 
Authored: Sat Nov 19 16:07:10 2016 +0800
Committer: Yang Li 
Committed: Sat Nov 19 16:07:10 2016 +0800

--
 .../kylin/cube/gridtable/TrimmedDimEnc.java |  2 +-
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|  2 +-
 .../apache/kylin/cube/kv/FuzzyMaskEncoder.java  |  4 +-
 .../org/apache/kylin/cube/kv/RowConstants.java  |  3 -
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|  4 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 33 +++--
 .../apache/kylin/cube/kv/RowKeyDecoderTest.java | 19 +++--
 .../apache/kylin/cube/kv/RowKeyEncoderTest.java | 42 +--
 .../gridtable/DimEncodingPreserveOrderTest.java | 18 +
 .../kylin/dimension/AbstractDateDimEnc.java | 17 +
 .../apache/kylin/dimension/BooleanDimEnc.java   | 19 ++---
 .../kylin/dimension/DictionaryDimEnc.java   |  7 +-
 .../kylin/dimension/DimensionEncoding.java  |  4 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  | 10 ++-
 .../kylin/dimension/FixedLenHexDimEnc.java  | 12 +--
 .../org/apache/kylin/dimension/IntDimEnc.java   | 12 +--
 .../apache/kylin/dimension/IntegerDimEnc.java   | 12 +--
 .../apache/kylin/dimension/IntegerDimEncV2.java | 12 +--
 .../kylin/dimension/OneMoreByteVLongDimEnc.java | 12 +--
 .../kylin/measure/topn/TopNMeasureType.java |  7 +-
 .../kylin/dimension/BooleanDimEncTest.java  |  6 +-
 .../kylin/dimension/FixedLenHexDimEncTest.java  |  6 +-
 .../apache/kylin/dimension/IntDimEncTest.java   |  6 +-
 .../kylin/dimension/IntegerDimEncTest.java  |  6 +-
 .../dimension/OneMoreByteVLongDimEncTest.java   |  6 +-
 .../engine/mr/steps/BaseCuboidMapperBase.java   | 77 +++-
 .../engine/mr/steps/HiveToBaseCuboidMapper.java |  9 +--
 .../coprocessor/CoprocessorProjector.java   |  4 +-
 .../common/coprocessor/FilterDecorator.java |  4 +-
 29 files changed, 122 insertions(+), 253 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e3ccd7/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
index c4bee8a..6d05668 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedDimEnc.java
@@ -44,7 +44,7 @@ public class TrimmedDimEnc extends DimensionEncoding {
 }
 
 @Override
-public void encode(byte[] value, int valueLen, byte[] output, int 
outputOffset) {
+public void encode(String valueStr, byte[] output, int outputOffset) {
 throw new UnsupportedOperationException();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e3ccd7/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
index 37b33aa..bfe6eb4 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
@@ -84,5 +84,5 @@ public abstract class AbstractRowKeyEncoder {
 
 abstract public byte[] encode(Map valueMap);
 
-abstract public byte[] encode(byte[][] values);
+abstract public byte[] encode(String[] values);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a6e3ccd7/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
index 94db94b..0cbb7d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/FuzzyMaskEncoder.java
@@ -77,8 +77,8 @@ public class FuzzyMaskEncoder extends RowKeyEncoder {
 }
 
 @Override
-protected void fillColumnValue(TblColRef column, int columnLen, byte[] 
value, int valueLen, byte[] outputValue, int outputValueOffset) {
-if (value