[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-02-26 Thread lidong
http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-02-26 Thread lidong
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24e0e9af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24e0e9af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24e0e9af

Branch: refs/heads/yang22-hbase102
Commit: 24e0e9afff255d74b3ff995cccd2a7c81000649c
Parents: d4dc6ee
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Feb 27 12:08:13 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-   

[4/4] kylin git commit: KYLIN-2233 Support HBase 1.0.2

2017-02-26 Thread lidong
KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7e5f6d65
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7e5f6d65
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7e5f6d65

Branch: refs/heads/yang22-hbase102
Commit: 7e5f6d65c1dd3d2b57d68f25789a0f2860ebb7f2
Parents: 24e0e9a
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu 
Committed: Mon Feb 27 12:14:01 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7e5f6d65/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dc23a2f..099316a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7e5f6d65/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/7e5f6d65/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of 

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-02-26 Thread lidong
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24e0e9af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24e0e9af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24e0e9af

Branch: refs/heads/yang22-cdh5.7
Commit: 24e0e9afff255d74b3ff995cccd2a7c81000649c
Parents: d4dc6ee
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Feb 27 12:08:13 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
- 

[1/4] kylin git commit: KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail [Forced Update!]

2017-02-26 Thread lidong
Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 9322c202f -> 5b4584fd4 (forced update)


KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4dc6ee0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4dc6ee0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4dc6ee0

Branch: refs/heads/yang22-cdh5.7
Commit: d4dc6ee09a1396d2c9408a6bc677746811e683e0
Parents: cdb33ca
Author: Hongbin Ma 
Authored: Sat Feb 25 16:36:19 2017 +0800
Committer: lidongsjtu 
Committed: Mon Feb 27 10:50:46 2017 +0800

--
 .../rest/util/CognosParenthesesEscape.java  | 83 
 .../rest/util/CognosParentesesEscapeTest.java   | 61 ++
 .../src/test/resources/query/cognos/query01.sql | 28 +++
 .../resources/query/cognos/query01.sql.expected | 28 +++
 4 files changed, 200 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d4dc6ee0/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
 
b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
new file mode 100644
index 000..33f4b63
--- /dev/null
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.util;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer;
+
+/**
+ * from (a join b on a.x = b.y) join c
+ * 
+ * similar in https://issues.apache.org/jira/browse/CALCITE-35
+ * 
+ * we'll find such pattern and remove the parentheses
+ */
+public class CognosParenthesesEscape implements IQueryTransformer {
+
+private static final String S0 = "\\s*";
+private static final String S1 = "\\s";
+private static final String SM = "\\s+";
+private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+";
+private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = 
TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?";
+private static final String JOIN = 
"(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per 
http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server
+private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + 
S0 + "=" + S0 + TABLE_OR_COLUMN_NAME;
+private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // (
+TABLE_NAME_WITH_OPTIONAL_ALIAS + // a
+JOIN + SM + // join
+TABLE_NAME_WITH_OPTIONAL_ALIAS + //b
+SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" 
+ // on a.x = b.y [and a.x2 = b.y2]
+S0 + "\\)";// )
+private static final Pattern PARENTTHESES_PATTERN = 
Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE);
+
+private static int identifierNum = 0;
+
+@Override
+public String transform(String sql) {
+Matcher m;
+List> matches = new LinkedList<>();
+while (true) {
+m = PARENTTHESES_PATTERN.matcher(sql);
+if (!m.find())
+break;
+
+String oneParentheses = m.group(0);
+String identifier = generateRandomName();
+matches.add(new Pair(identifier, 
oneParentheses.substring(1, oneParentheses.length() - 1)));
+sql = sql.substring(0, m.start()) + identifier + 
sql.substring(m.end());
+}
+
+for (int i = matches.size() - 1; i >= 0; i--) {
+sql = sql.replaceAll(matches.get(i).getKey(), 

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2017-02-26 Thread lidong
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5b4584fd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5b4584fd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5b4584fd

Branch: refs/heads/yang22-cdh5.7
Commit: 5b4584fd4e5c6c402832c2c26c18b7a1cbe0391f
Parents: 24e0e9a
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Mon Feb 27 12:12:02 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5b4584fd/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/5b4584fd/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5b4584fd/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[1/3] kylin git commit: KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail [Forced Update!]

2017-02-26 Thread lidong
Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 1c57f3e0d -> 24e0e9aff (forced update)


KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4dc6ee0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4dc6ee0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4dc6ee0

Branch: refs/heads/yang22-hbase1.x
Commit: d4dc6ee09a1396d2c9408a6bc677746811e683e0
Parents: cdb33ca
Author: Hongbin Ma 
Authored: Sat Feb 25 16:36:19 2017 +0800
Committer: lidongsjtu 
Committed: Mon Feb 27 10:50:46 2017 +0800

--
 .../rest/util/CognosParenthesesEscape.java  | 83 
 .../rest/util/CognosParentesesEscapeTest.java   | 61 ++
 .../src/test/resources/query/cognos/query01.sql | 28 +++
 .../resources/query/cognos/query01.sql.expected | 28 +++
 4 files changed, 200 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d4dc6ee0/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
 
b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
new file mode 100644
index 000..33f4b63
--- /dev/null
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.util;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer;
+
+/**
+ * from (a join b on a.x = b.y) join c
+ * 
+ * similar in https://issues.apache.org/jira/browse/CALCITE-35
+ * 
+ * we'll find such pattern and remove the parentheses
+ */
+public class CognosParenthesesEscape implements IQueryTransformer {
+
+private static final String S0 = "\\s*";
+private static final String S1 = "\\s";
+private static final String SM = "\\s+";
+private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+";
+private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = 
TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?";
+private static final String JOIN = 
"(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per 
http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server
+private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + 
S0 + "=" + S0 + TABLE_OR_COLUMN_NAME;
+private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // (
+TABLE_NAME_WITH_OPTIONAL_ALIAS + // a
+JOIN + SM + // join
+TABLE_NAME_WITH_OPTIONAL_ALIAS + //b
+SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" 
+ // on a.x = b.y [and a.x2 = b.y2]
+S0 + "\\)";// )
+private static final Pattern PARENTTHESES_PATTERN = 
Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE);
+
+private static int identifierNum = 0;
+
+@Override
+public String transform(String sql) {
+Matcher m;
+List> matches = new LinkedList<>();
+while (true) {
+m = PARENTTHESES_PATTERN.matcher(sql);
+if (!m.find())
+break;
+
+String oneParentheses = m.group(0);
+String identifier = generateRandomName();
+matches.add(new Pair(identifier, 
oneParentheses.substring(1, oneParentheses.length() - 1)));
+sql = sql.substring(0, m.start()) + identifier + 
sql.substring(m.end());
+}
+
+for (int i = matches.size() - 1; i >= 0; i--) {
+sql = sql.replaceAll(matches.get(i).getKey(), 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-02-26 Thread lidong
http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/24e0e9af/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

svn commit: r1784444 - in /kylin/site: ./ blog/ blog/2017/02/23/ blog/2017/02/23/by-layer-spark-cubing/ images/blog/

2017-02-26 Thread lidong
Author: lidong
Date: Sun Feb 26 13:59:58 2017
New Revision: 178

URL: http://svn.apache.org/viewvc?rev=178=rev
Log:
add post for spark cubing

Added:
kylin/site/blog/2017/02/23/
kylin/site/blog/2017/02/23/by-layer-spark-cubing/
kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html
kylin/site/images/blog/spark-cubing-layer.png   (with props)
kylin/site/images/blog/spark-dag.png   (with props)
kylin/site/images/blog/spark-mr-layer.png   (with props)
kylin/site/images/blog/spark-mr-performance.png   (with props)
Modified:
kylin/site/blog/index.html
kylin/site/feed.xml

Added: kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html?rev=178=auto
==
--- kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html (added)
+++ kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html Sun Feb 26 
13:59:58 2017
@@ -0,0 +1,307 @@
+
+
+
+   
+
+
+  
+  
+  
+
+  Apache Kylin | By-layer Spark Cubing
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+  
+  
+
+  http://kylin.apache.org/blog/2017/02/23/by-layer-spark-cubing/;>
+  http://kylin.apache.org/feed.xml; />
+
+
+
+  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new 
Date();a=s.createElement(o),
+  
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+  //oringal tracker for kylin.io
+  ga('create', 'UA-55534813-1', 'auto');
+  //new tracker for kylin.apache.org
+  ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+  ga('send', 'pageview');
+  ga('toplevel.send', 'pageview');
+
+
+
+
+ 
+ 
+
+
+   
+   
+
+
+  
+  
+  Apache Kylin™ Extreme OLAP Engine for Big 
Data 
+
+  
+  
+
+  
+  
+  
+
+
+
+  
+Toggle navigation
+
+
+
+  
+ 
+
+
+
+
+  
+ Home
+  Docs
+  Download
+  Community
+  Development
+  Blog
+  中文版  
+  https://twitter.com/apachekylin; target="_blank" 
class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" >
+  https://github.com/apache/kylin; target="_blank" 
class="fa fa-github-alt fa-lg" title="Github: apache/kylin" >  
+  https://www.facebook.com/kylinio; target="_blank" 
class="fa fa-facebook fa-lg" title="Facebook: kylin.io" >   
+
+
+  
+
+ 
+
+   
+   
+   
+   Apache Kylin™ 
Technical Blog
+   
+   
+
+   
+   
+
+   
+
+
+
+  
+By-layer Spark Cubing
+Feb 23, 2017 • Shaofeng Shi
+  
+
+  
+Before v2.0, Apache Kylin uses Hadoop MapReduce as the framework to 
build Cubes over huge dataset. The MapReduce framework is simple, stable and 
can fulfill Kylin’s need very well except the performance. In order to get 
better performance, we introduced the “fast cubing” algorithm in Kylin 
v1.5, tries to do as much as possible aggregations at map side within memory, 
so to avoid the disk and network I/O; but not all data models can benefit from 
it, and it still runs on MR which means on-disk sorting and shuffling.
+
+Now Spark comes; Apache Spark is an open-source cluster-computing 
framework, which provides programmers with an application programming interface 
centered on a data structure called RDD; it runs in-memory on the cluster, this 
makes repeated access to the same data much faster. Spark provides flexible and 
fancy APIs. You are not tied to Hadoop’s MapReduce two-stage paradigm.
+
+Before introducing how calculate Cube with Spark, let’s see how Kylin do 
that with MR; Figure 1 illustrates how a 4-dimension Cube get calculated with 
the classic “by-layer” algorithm: the first round MR aggregates the base 
(4-D) cuboid from source data; the second MR aggregates on the base cuboid to 
get the 3-D cuboids; With N+1 round MR all layers’ cuboids get calculated.
+
+
+
+The “by-layer” Cubing divides a big task into a couple steps, and each 
step bases on the previous step’s output, so it can reuse the previous 
calculation and also avoid calculating from very beginning when there is a 
failure in between. These makes it as a reliable algorithm. When moving to 
Spark, we decide to keep this algorithm, that’s why we call this feature as 
“By layer Spark Cubing”.
+
+As we know, RDD (Resilient Distributed Dataset) is a basic concept in 
Spark. A collection of N-Dimension cuboids can be well described as an RDD, a 
N-Dimension Cube will have N+1 RDD. These RDDs have the