date:20170410

kylin git commit: KYLIN-2539 revert unnessary changes

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 794ef4d43 -> a36288142


KYLIN-2539 revert unnessary changes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a3628814
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a3628814
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a3628814

Branch: refs/heads/master
Commit: a36288142f30cad931881107872adb4ba0a574a3
Parents: 794ef4d
Author: Hongbin Ma 
Authored: Tue Apr 11 13:55:11 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 13:55:11 2017 +0800

--
 examples/test_case_data/sandbox/kylin.properties | 2 +-
 .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a3628814/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 2a8f617..684b4dd 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -39,7 +39,7 @@ kylin.source.hive.client=cli
 ### STORAGE ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_ci_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
 
 
 # The storage for final cube file in hbase

http://git-wip-us.apache.org/repos/asf/kylin/blob/a3628814/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 271b8ff..cdaa10b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -134,11 +134,11 @@ public class ITKylinQueryTest extends KylinTestBase {
 @Test
 public void testSingleRunQuery() throws Exception {
 
-String queryFileName = getQueryFolderPrefix() + 
"src/test/resources/query/temp/query00.sql";
+String queryFileName = getQueryFolderPrefix() + 
"src/test/resources/query/sql_verifyCount/query03.sql";
 
 File sqlFile = new File(queryFileName);
 if (sqlFile.exists()) {
-runSQL(sqlFile, true, true);
+//runSQL(sqlFile, true, true);
 runSQL(sqlFile, true, false);
 }
 }

kylin git commit: KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master bd3b84402 -> 794ef4d43


KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/794ef4d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/794ef4d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/794ef4d4

Branch: refs/heads/master
Commit: 794ef4d4375d03f490b0d1c5da7b5ce1a9dac75e
Parents: bd3b844
Author: Hongbin Ma 
Authored: Tue Apr 11 13:49:12 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 13:49:17 2017 +0800

--
 .../test_case_data/sandbox/kylin.properties |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  4 +--
 .../kylin/query/relnode/OLAPFilterRel.java  | 28 +---
 3 files changed, 16 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/794ef4d4/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 684b4dd..2a8f617 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -39,7 +39,7 @@ kylin.source.hive.client=cli
 ### STORAGE ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
+kylin.metadata.url=kylin_ci_instance@hbase
 
 
 # The storage for final cube file in hbase

http://git-wip-us.apache.org/repos/asf/kylin/blob/794ef4d4/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index cdaa10b..271b8ff 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -134,11 +134,11 @@ public class ITKylinQueryTest extends KylinTestBase {
 @Test
 public void testSingleRunQuery() throws Exception {
 
-String queryFileName = getQueryFolderPrefix() + 
"src/test/resources/query/sql_verifyCount/query03.sql";
+String queryFileName = getQueryFolderPrefix() + 
"src/test/resources/query/temp/query00.sql";
 
 File sqlFile = new File(queryFileName);
 if (sqlFile.exists()) {
-//runSQL(sqlFile, true, true);
+runSQL(sqlFile, true, true);
 runSQL(sqlFile, true, false);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/794ef4d4/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 3fba1ee..0ee9204 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -20,7 +20,6 @@ package org.apache.kylin.query.relnode;
 
 import java.util.GregorianCalendar;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -70,6 +69,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 /**
  */
@@ -78,13 +78,10 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 private static class TupleFilterVisitor extends 
RexVisitorImpl {
 
 private final ColumnRowType inputRowType;
-private final OLAPContext context;
-private final Set columnsInFilter = new HashSet<>();
 
-public TupleFilterVisitor(ColumnRowType inputRowType, OLAPContext 
context) {
+public TupleFilterVisitor(ColumnRowType inputRowType) {
 super(true);
 this.inputRowType = inputRowType;
-this.context = context;
 }
 
 @Override
@@ -229,10 +226,6 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 @Override
 public TupleFilter visitInputRef(RexInputRef inputRef) {
 TblColRef column = 
inputRowType.getColumnByIndex(inputRef.getIndex());
-if (!column.isInnerColumn()) {
-context.allColumns.add(column);
-columnsInFilter.add(column);
-}
 ColumnTupleFilter filter = new ColumnTupleFilter(column);
 return filter;
 }
@@ -322,15 +315,20 @@ public class OLAPFilterRel

[4/4] kylin git commit: KYLIN-2540 remove unused test

2017-04-10 Thread lidong

KYLIN-2540 remove unused test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdda7610
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdda7610
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdda7610

Branch: refs/heads/2.0.x
Commit: fdda7610b6cf4a1477ecc2561d55b15b2c264f1e
Parents: c4b1d4a
Author: lidongsjtu 
Authored: Mon Apr 10 22:22:19 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 13:01:35 2017 +0800

--
 .../test/java/org/apache/kylin/rest/util/QueryUtilTest.java| 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fdda7610/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
--
diff --git 
a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java 
b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
index c00cd3f..dcd8c98 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
@@ -51,12 +51,6 @@ public class QueryUtilTest extends LocalFileMetadataTestCase 
{
 String s = QueryUtil.massageSql(sqlRequest);
 Assert.assertEquals("select ( date '2001-09-28' + interval '2' 
month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' 
month)", s);
 }
-{
-SQLRequest sqlRequest = new SQLRequest();
-sqlRequest.setSql("select 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 ()");
-String s = QueryUtil.massageSql(sqlRequest);
-Assert.assertEquals("select {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } ()", s);
-}
 }
 
 @Test

[3/4] kylin git commit: KYLIN-2539 bug fix

2017-04-10 Thread lidong

KYLIN-2539 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c4b1d4a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c4b1d4a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c4b1d4a5

Branch: refs/heads/2.0.x
Commit: c4b1d4a5a4e11f8c50b9957e38eba71d18bc6143
Parents: fac8285
Author: lidongsjtu 
Authored: Tue Apr 11 10:46:22 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 13:01:28 2017 +0800

--
 .../apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 4 
 .../main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java | 5 +
 2 files changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c4b1d4a5/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index abac33a..08a3843 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,7 +38,6 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -149,9 +148,6 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 // set query deadline
 context.setDeadline(cubeInstance);
 
-// optimize the filter, the optimization has to be segment-irrelevant
-new FilterOptimizeTransformer().transform(filterD);
-
 logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, 
filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), 
cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), 
context.isNeedStorageAggregation());
 
 return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, 
metrics, filterD, context);

http://git-wip-us.apache.org/repos/asf/kylin/blob/c4b1d4a5/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index d3d78a0..3fba1ee 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -60,6 +60,7 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.DynamicTupleFilter;
 import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -325,6 +326,10 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 TupleFilter filter = this.condition.accept(visitor);
 
 context.filter = TupleFilter.and(context.filter, filter);
+
+// optimize the filter, the optimization has to be segment-irrelevant
+new FilterOptimizeTransformer().transform(context.filter);
+
 context.filterColumns.addAll(visitor.columnsInFilter);
 }

[1/4] kylin git commit: KYLIN-2541 fix sql missing in logs for beeline

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/2.0.x 273a40989 -> fdda7610b


KYLIN-2541 fix sql missing in logs for beeline


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c538b1a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c538b1a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c538b1a

Branch: refs/heads/2.0.x
Commit: 0c538b1a041f2dd7e7ba65f83bcb95a58d9dbc2a
Parents: 273a409
Author: lidongsjtu 
Authored: Tue Apr 11 12:45:48 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 13:01:02 2017 +0800

--
 .../apache/kylin/common/util/HiveCmdBuilder.java| 16 
 .../kylin/common/util/HiveCmdBuilderTest.java   |  1 +
 2 files changed, 13 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c538b1a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 5942ba9..855ee48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -62,8 +62,9 @@ public class HiveCmdBuilder {
 break;
 case BEELINE:
 BufferedWriter bw = null;
+File tmpHql = null;
 try {
-File tmpHql = File.createTempFile("beeline_", ".hql");
+tmpHql = File.createTempFile("beeline_", ".hql");
 bw = new BufferedWriter(new FileWriter(tmpHql));
 for (String statement : statements) {
 bw.write(statement);
@@ -77,13 +78,20 @@ public class HiveCmdBuilder {
 buf.append(tmpHql.getAbsolutePath());
 buf.append(";exit $ret_code");
 
-if (logger.isDebugEnabled()) {
-logger.debug("The SQL to execute in beeline: \n" + 
FileUtils.readFileToString(tmpHql, Charset.defaultCharset()));
-}
 } catch (IOException e) {
 throw new RuntimeException(e);
 } finally {
 IOUtils.closeQuietly(bw);
+
+if (tmpHql != null && logger.isDebugEnabled()) {
+String hql = null;
+try {
+hql = FileUtils.readFileToString(tmpHql, 
Charset.defaultCharset());
+} catch (IOException e) {
+// ignore
+}
+logger.debug("The SQL to execute in beeline: \n" + hql);
+}
 }
 break;
 default:

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c538b1a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index 56a64c8..3bc34b4 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -34,6 +34,7 @@ public class HiveCmdBuilderTest {
 
 @Before
 public void setup() {
+System.setProperty("log4j.configuration", 
"file:../build/conf/kylin-tools-log4j.properties");
 System.setProperty("KYLIN_CONF", 
LocalFileMetadataTestCase.LOCALMETA_TEST_DATA);
 }

[2/4] kylin git commit: KYLIN-2540 fix concat cascade issue

2017-04-10 Thread lidong

KYLIN-2540 fix concat cascade issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fac8285a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fac8285a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fac8285a

Branch: refs/heads/2.0.x
Commit: fac8285acf7db2472b95688fe4a85ca270293c99
Parents: 0c538b1
Author: lidongsjtu 
Authored: Mon Apr 10 21:50:32 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 13:01:21 2017 +0800

--
 build/conf/kylin.properties |  1 +
 .../test_case_data/sandbox/kylin.properties |  1 +
 .../src/test/resources/query/sql/query104.sql   | 31 
 .../org/apache/kylin/query/udf/ConcatUDF.java   | 28 ++
 .../org/apache/kylin/rest/util/QueryUtil.java   | 15 --
 5 files changed, 61 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 6d8422e..a2273d5 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -164,6 +164,7 @@ kylin.snapshot.max-mb=300
 kylin.query.max-scan-bytes=0
 
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 
 kylin.query.cache-enabled=true
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 1674d82..684b4dd 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -108,6 +108,7 @@ kylin.storage.hbase.max-region-count=5
 kylin.storage.hbase.hfile-size-gb=2
 
 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
 
 # for test

http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/kylin-it/src/test/resources/query/sql/query104.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query104.sql 
b/kylin-it/src/test/resources/query/sql/query104.sql
new file mode 100644
index 000..bb85664
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query104.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select concat(meta_categ_name, lstg_format_name) as c1, 
concat(meta_categ_name, 'CONST') as c2, concat(meta_categ_name, 
concat(test_sites.site_name, lstg_format_name)) as c3, count(1) as cnt, 
sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name 
not in ('Crafts','Computers'))
+ group by concat(meta_categ_name, lstg_format_name), concat(meta_categ_name, 
'CONST'), concat(meta_categ_name, concat(test_sites.site_name, 
lstg_format_name))
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java 
b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
new file mode 100644
index 000..9c0da53
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
@@

kylin git commit: KYLIN-2541 fix sql missing in logs for beeline

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master 9532bf2b0 -> bd3b84402


KYLIN-2541 fix sql missing in logs for beeline


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd3b8440
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd3b8440
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd3b8440

Branch: refs/heads/master
Commit: bd3b84402a8f75f02d7f5427d3261ee7c880a5d0
Parents: 9532bf2
Author: lidongsjtu 
Authored: Tue Apr 11 12:45:48 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 12:45:52 2017 +0800

--
 .../apache/kylin/common/util/HiveCmdBuilder.java| 16 
 .../kylin/common/util/HiveCmdBuilderTest.java   |  1 +
 2 files changed, 13 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bd3b8440/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 5942ba9..855ee48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -62,8 +62,9 @@ public class HiveCmdBuilder {
 break;
 case BEELINE:
 BufferedWriter bw = null;
+File tmpHql = null;
 try {
-File tmpHql = File.createTempFile("beeline_", ".hql");
+tmpHql = File.createTempFile("beeline_", ".hql");
 bw = new BufferedWriter(new FileWriter(tmpHql));
 for (String statement : statements) {
 bw.write(statement);
@@ -77,13 +78,20 @@ public class HiveCmdBuilder {
 buf.append(tmpHql.getAbsolutePath());
 buf.append(";exit $ret_code");
 
-if (logger.isDebugEnabled()) {
-logger.debug("The SQL to execute in beeline: \n" + 
FileUtils.readFileToString(tmpHql, Charset.defaultCharset()));
-}
 } catch (IOException e) {
 throw new RuntimeException(e);
 } finally {
 IOUtils.closeQuietly(bw);
+
+if (tmpHql != null && logger.isDebugEnabled()) {
+String hql = null;
+try {
+hql = FileUtils.readFileToString(tmpHql, 
Charset.defaultCharset());
+} catch (IOException e) {
+// ignore
+}
+logger.debug("The SQL to execute in beeline: \n" + hql);
+}
 }
 break;
 default:

http://git-wip-us.apache.org/repos/asf/kylin/blob/bd3b8440/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index 56a64c8..3bc34b4 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -34,6 +34,7 @@ public class HiveCmdBuilderTest {
 
 @Before
 public void setup() {
+System.setProperty("log4j.configuration", 
"file:../build/conf/kylin-tools-log4j.properties");
 System.setProperty("KYLIN_CONF", 
LocalFileMetadataTestCase.LOCALMETA_TEST_DATA);
 }

Jenkins build is back to stable : Kylin-Master-JDK-1.7 #395

2017-04-10 Thread Apache Jenkins Server

See

Jenkins build became unstable: Kylin-Master-JDK-1.7 #394

2017-04-10 Thread Apache Jenkins Server

See

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ce546fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ce546fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ce546fe

Branch: refs/heads/yang22-hbase102
Commit: 8ce546fe9923919f5c21e75116f0884b2119dbf6
Parents: c2cdce8
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:53:03 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/5] kylin git commit: KYLIN-2539 bug fix [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 bc8cbbd67 -> 2206543ec (forced update)


KYLIN-2539 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba94580f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba94580f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba94580f

Branch: refs/heads/yang22-hbase102
Commit: ba94580f993ece8632026eeb5f9461c456664987
Parents: 35b7a29
Author: Hongbin Ma 
Authored: Tue Apr 11 10:19:56 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 10:19:56 2017 +0800

--
 .../apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 2 --
 .../main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java | 5 -
 2 files changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ba94580f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 575ca22..f841365 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -132,8 +132,6 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 logger.info("Skip cube segment {} because its input record is 
0", cubeSeg);
 continue;
 }
-// optimize the filter, the optimization has to be 
segment-irrelevant
-new FilterOptimizeTransformer().transform(filterD);
 
 scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context);
 scanners.add(scanner);

http://git-wip-us.apache.org/repos/asf/kylin/blob/ba94580f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 1981c10..c826ff3 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -59,10 +59,11 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.DynamicTupleFilter;
 import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.function.Functions;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -319,6 +320,8 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 
 TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType, context);
 context.filter = this.condition.accept(visitor);
+// optimize the filter, the optimization has to be segment-irrelevant
+new FilterOptimizeTransformer().transform(context.filter);
 
 context.filterColumns = collectColumns(context.filter);
 }

[5/5] kylin git commit: KYLIN-2233 Support HBase 1.0.2

2017-04-10 Thread lidong

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2206543e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2206543e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2206543e

Branch: refs/heads/yang22-hbase102
Commit: 2206543ec06c70be716d18b7e834ee52c2e884f9
Parents: 8ce546f
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:57:40 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2206543e/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dc23a2f..099316a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2206543e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/2206543e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of

[2/5] kylin git commit: KYLIN-2539 fix checkstyle

2017-04-10 Thread lidong

KYLIN-2539 fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2cdce8a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2cdce8a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2cdce8a

Branch: refs/heads/yang22-hbase102
Commit: c2cdce8a3efbd8a6bfb3c48ee4a2a6a936c53992
Parents: ba94580
Author: lidongsjtu 
Authored: Tue Apr 11 10:48:02 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:48:02 2017 +0800

--
 .../org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2cdce8a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index f841365..2c25ced 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,7 +38,6 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ce546fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ce546fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ce546fe

Branch: refs/heads/yang22-cdh5.7
Commit: 8ce546fe9923919f5c21e75116f0884b2119dbf6
Parents: c2cdce8
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:53:03 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[2/5] kylin git commit: KYLIN-2539 fix checkstyle

2017-04-10 Thread lidong

KYLIN-2539 fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2cdce8a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2cdce8a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2cdce8a

Branch: refs/heads/yang22-cdh5.7
Commit: c2cdce8a3efbd8a6bfb3c48ee4a2a6a936c53992
Parents: ba94580
Author: lidongsjtu 
Authored: Tue Apr 11 10:48:02 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:48:02 2017 +0800

--
 .../org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2cdce8a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index f841365..2c25ced 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,7 +38,6 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;

[5/5] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2017-04-10 Thread lidong

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/014f72d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/014f72d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/014f72d8

Branch: refs/heads/yang22-cdh5.7
Commit: 014f72d83797e1106d73972b3c496599056a4561
Parents: 8ce546f
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Tue Apr 11 10:55:25 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/014f72d8/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/014f72d8/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/014f72d8/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[1/5] kylin git commit: KYLIN-2539 bug fix [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 2a0ef6a3b -> 014f72d83 (forced update)


KYLIN-2539 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba94580f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba94580f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba94580f

Branch: refs/heads/yang22-cdh5.7
Commit: ba94580f993ece8632026eeb5f9461c456664987
Parents: 35b7a29
Author: Hongbin Ma 
Authored: Tue Apr 11 10:19:56 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 10:19:56 2017 +0800

--
 .../apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 2 --
 .../main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java | 5 -
 2 files changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ba94580f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 575ca22..f841365 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -132,8 +132,6 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 logger.info("Skip cube segment {} because its input record is 
0", cubeSeg);
 continue;
 }
-// optimize the filter, the optimization has to be 
segment-irrelevant
-new FilterOptimizeTransformer().transform(filterD);
 
 scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context);
 scanners.add(scanner);

http://git-wip-us.apache.org/repos/asf/kylin/blob/ba94580f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 1981c10..c826ff3 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -59,10 +59,11 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.DynamicTupleFilter;
 import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.function.Functions;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -319,6 +320,8 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 
 TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType, context);
 context.filter = this.condition.accept(visitor);
+// optimize the filter, the optimization has to be segment-irrelevant
+new FilterOptimizeTransformer().transform(context.filter);
 
 context.filterColumns = collectColumns(context.filter);
 }

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ce546fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ce546fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ce546fe

Branch: refs/heads/yang22-hbase1.x
Commit: 8ce546fe9923919f5c21e75116f0884b2119dbf6
Parents: c2cdce8
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:53:03 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/4] kylin git commit: KYLIN-2539 bug fix [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x b47c8ab8b -> 8ce546fe9 (forced update)


KYLIN-2539 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba94580f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba94580f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba94580f

Branch: refs/heads/yang22-hbase1.x
Commit: ba94580f993ece8632026eeb5f9461c456664987
Parents: 35b7a29
Author: Hongbin Ma 
Authored: Tue Apr 11 10:19:56 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 10:19:56 2017 +0800

--
 .../apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 2 --
 .../main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java | 5 -
 2 files changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ba94580f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 575ca22..f841365 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -132,8 +132,6 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 logger.info("Skip cube segment {} because its input record is 
0", cubeSeg);
 continue;
 }
-// optimize the filter, the optimization has to be 
segment-irrelevant
-new FilterOptimizeTransformer().transform(filterD);
 
 scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context);
 scanners.add(scanner);

http://git-wip-us.apache.org/repos/asf/kylin/blob/ba94580f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 1981c10..c826ff3 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -59,10 +59,11 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.DynamicTupleFilter;
 import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.function.Functions;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -319,6 +320,8 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 
 TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType, context);
 context.filter = this.condition.accept(visitor);
+// optimize the filter, the optimization has to be segment-irrelevant
+new FilterOptimizeTransformer().transform(context.filter);
 
 context.filterColumns = collectColumns(context.filter);
 }

[2/4] kylin git commit: KYLIN-2539 fix checkstyle

2017-04-10 Thread lidong

KYLIN-2539 fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2cdce8a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2cdce8a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2cdce8a

Branch: refs/heads/yang22-hbase1.x
Commit: c2cdce8a3efbd8a6bfb3c48ee4a2a6a936c53992
Parents: ba94580
Author: lidongsjtu 
Authored: Tue Apr 11 10:48:02 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:48:02 2017 +0800

--
 .../org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2cdce8a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index f841365..2c25ced 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,7 +38,6 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce546fe/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[1/2] kylin git commit: KYLIN-2540 fix concat cascade issue

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master 177079d10 -> 86645af9d


KYLIN-2540 fix concat cascade issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e248e85
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e248e85
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e248e85

Branch: refs/heads/master
Commit: 6e248e85fbf35440f7e59b5383c99ad3f80079b7
Parents: 177079d
Author: lidongsjtu 
Authored: Mon Apr 10 21:50:32 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 21:50:32 2017 +0800

--
 build/conf/kylin.properties |  1 +
 .../test_case_data/sandbox/kylin.properties |  1 +
 .../src/test/resources/query/sql/query104.sql   | 31 
 .../org/apache/kylin/query/udf/ConcatUDF.java   | 28 ++
 .../org/apache/kylin/rest/util/QueryUtil.java   | 15 --
 5 files changed, 61 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6e248e85/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 6d8422e..a2273d5 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -164,6 +164,7 @@ kylin.snapshot.max-mb=300
 kylin.query.max-scan-bytes=0
 
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 
 kylin.query.cache-enabled=true
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e248e85/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 1674d82..684b4dd 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -108,6 +108,7 @@ kylin.storage.hbase.max-region-count=5
 kylin.storage.hbase.hfile-size-gb=2
 
 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
 
 # for test

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e248e85/kylin-it/src/test/resources/query/sql/query104.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query104.sql 
b/kylin-it/src/test/resources/query/sql/query104.sql
new file mode 100644
index 000..bb85664
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query104.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select concat(meta_categ_name, lstg_format_name) as c1, 
concat(meta_categ_name, 'CONST') as c2, concat(meta_categ_name, 
concat(test_sites.site_name, lstg_format_name)) as c3, count(1) as cnt, 
sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name 
not in ('Crafts','Computers'))
+ group by concat(meta_categ_name, lstg_format_name), concat(meta_categ_name, 
'CONST'), concat(meta_categ_name, concat(test_sites.site_name, 
lstg_format_name))
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e248e85/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java 
b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
new file mode 100644
index 000..9c0da53
---

[2/2] kylin git commit: KYLIN-2539 bug fix

2017-04-10 Thread lidong

KYLIN-2539 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/86645af9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/86645af9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/86645af9

Branch: refs/heads/master
Commit: 86645af9ddec2289b330bebb3593c673e8ab2ff6
Parents: 6e248e8
Author: lidongsjtu 
Authored: Tue Apr 11 10:46:22 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:46:22 2017 +0800

--
 .../apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 4 
 .../main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java | 5 +
 2 files changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/86645af9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index abac33a..08a3843 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,7 +38,6 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -149,9 +148,6 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 // set query deadline
 context.setDeadline(cubeInstance);
 
-// optimize the filter, the optimization has to be segment-irrelevant
-new FilterOptimizeTransformer().transform(filterD);
-
 logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, 
filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), 
cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), 
context.isNeedStorageAggregation());
 
 return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, 
metrics, filterD, context);

http://git-wip-us.apache.org/repos/asf/kylin/blob/86645af9/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index d3d78a0..3fba1ee 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -60,6 +60,7 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.DynamicTupleFilter;
 import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -325,6 +326,10 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 TupleFilter filter = this.condition.accept(visitor);
 
 context.filter = TupleFilter.and(context.filter, filter);
+
+// optimize the filter, the optimization has to be segment-irrelevant
+new FilterOptimizeTransformer().transform(context.filter);
+
 context.filterColumns.addAll(visitor.columnsInFilter);
 }

kylin git commit: KYLIN-2539 fix checkstyle

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22 ba94580f9 -> c2cdce8a3


KYLIN-2539 fix checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2cdce8a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2cdce8a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2cdce8a

Branch: refs/heads/yang22
Commit: c2cdce8a3efbd8a6bfb3c48ee4a2a6a936c53992
Parents: ba94580
Author: lidongsjtu 
Authored: Tue Apr 11 10:48:02 2017 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 10:48:02 2017 +0800

--
 .../org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2cdce8a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index f841365..2c25ced 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,7 +38,6 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;

kylin git commit: KYLIN-2539 bug fix

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang22 35b7a2914 -> ba94580f9


KYLIN-2539 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba94580f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba94580f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba94580f

Branch: refs/heads/yang22
Commit: ba94580f993ece8632026eeb5f9461c456664987
Parents: 35b7a29
Author: Hongbin Ma 
Authored: Tue Apr 11 10:19:56 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 10:19:56 2017 +0800

--
 .../apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 2 --
 .../main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java | 5 -
 2 files changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ba94580f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 575ca22..f841365 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -132,8 +132,6 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 logger.info("Skip cube segment {} because its input record is 
0", cubeSeg);
 continue;
 }
-// optimize the filter, the optimization has to be 
segment-irrelevant
-new FilterOptimizeTransformer().transform(filterD);
 
 scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context);
 scanners.add(scanner);

http://git-wip-us.apache.org/repos/asf/kylin/blob/ba94580f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 1981c10..c826ff3 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -59,10 +59,11 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.DynamicTupleFilter;
 import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.function.Functions;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -319,6 +320,8 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 
 TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType, context);
 context.filter = this.condition.accept(visitor);
+// optimize the filter, the optimization has to be segment-irrelevant
+new FilterOptimizeTransformer().transform(context.filter);
 
 context.filterColumns = collectColumns(context.filter);
 }

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b47c8ab8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b47c8ab8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b47c8ab8

Branch: refs/heads/yang22-hbase102
Commit: b47c8ab8be43e8f1ac78de0e7daeea2f880cfb7b
Parents: 35b7a29
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 23:15:31 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2

2017-04-10 Thread lidong

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc8cbbd6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc8cbbd6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc8cbbd6

Branch: refs/heads/yang22-hbase102
Commit: bc8cbbd671f176c349f217dd5345dba8873210e5
Parents: b47c8ab
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 23:20:25 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bc8cbbd6/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dc23a2f..099316a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc8cbbd6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc8cbbd6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 f71645e94 -> bc8cbbd67 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b47c8ab8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b47c8ab8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b47c8ab8

Branch: refs/heads/yang22-cdh5.7
Commit: b47c8ab8be43e8f1ac78de0e7daeea2f880cfb7b
Parents: 35b7a29
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 23:15:31 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 c7678a9c3 -> 2a0ef6a3b (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b47c8ab8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b47c8ab8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b47c8ab8

Branch: refs/heads/yang22-hbase1.x
Commit: b47c8ab8be43e8f1ac78de0e7daeea2f880cfb7b
Parents: 35b7a29
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 23:15:31 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x f245d675a -> b47c8ab8b (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b47c8ab8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();

[3/6] kylin git commit: KYLIN-2540 remove unused test

2017-04-10 Thread lidong

KYLIN-2540 remove unused test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/35b7a291
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/35b7a291
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/35b7a291

Branch: refs/heads/yang22-hbase102
Commit: 35b7a29148222c425644c897d6129e123bbb01d7
Parents: 1c3296f
Author: lidongsjtu 
Authored: Mon Apr 10 22:22:19 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:22:19 2017 +0800

--
 .../test/java/org/apache/kylin/rest/util/QueryUtilTest.java| 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/35b7a291/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
--
diff --git 
a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java 
b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
index c00cd3f..dcd8c98 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
@@ -51,12 +51,6 @@ public class QueryUtilTest extends LocalFileMetadataTestCase 
{
 String s = QueryUtil.massageSql(sqlRequest);
 Assert.assertEquals("select ( date '2001-09-28' + interval '2' 
month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' 
month)", s);
 }
-{
-SQLRequest sqlRequest = new SQLRequest();
-sqlRequest.setSql("select 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 ()");
-String s = QueryUtil.massageSql(sqlRequest);
-Assert.assertEquals("select {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } ()", s);
-}
 }
 
 @Test

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[5/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f245d675
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f245d675
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f245d675

Branch: refs/heads/yang22-hbase102
Commit: f245d675a34c482a2ce05553c0173c394302e03f
Parents: 35b7a29
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:24:27 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/6] kylin git commit: KYLIN-2539 Useless filter dimension will impact cuboid selection. [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 a055a4cd9 -> f71645e94 (forced update)


KYLIN-2539 Useless filter dimension will impact cuboid selection.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d4d741b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d4d741b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d4d741b

Branch: refs/heads/yang22-hbase102
Commit: 9d4d741b254d08c9a5f30921478ebf956c09b9e9
Parents: ea778d2
Author: Hongbin Ma 
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 10 21:31:50 2017 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 14 +++
 .../filter/FilterOptimizeTransformer.java   | 89 
 .../gtrecord/GTCubeStorageQueryBase.java|  5 +-
 3 files changed, 107 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..f7cfb6a 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -215,6 +215,20 @@ public class CompareTupleFilter extends TupleFilter {
 return ((function != null && function.isEvaluable()) || column != 
null) && !conditionValues.isEmpty();
 }
 
+public boolean alwaysReturnTrue() {
+// 1 = 1
+if (this.operator == FilterOperatorEnum.EQ) {
+if (this.children != null && this.children.size() == 2 && //
+this.children.get(0) instanceof ConstantTupleFilter && //
+this.children.get(1) instanceof ConstantTupleFilter && //
+((ConstantTupleFilter) 
this.children.get(0)).getValues().equals(((ConstantTupleFilter) 
this.children.get(1)).getValues())) {
+return true;
+}
+}
+
+return false;
+}
+
 @SuppressWarnings({ "unchecked", "rawtypes" })
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 000..a3bbad6
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * 
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+public static final Logger logger = 
LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+@Override
+public TupleFilter transform(TupleFilter tupleFilter) {
+TupleFilter translated = null;
+if (tupleFilter instanceof CompareTupleFilter) {
+//normal case
+translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) 
tupleFilter);
+} else if (tupleFilter instanceof LogicalTupleFilter) {
+@SuppressWarnings("unchecked")
+ListIterator childIterator = 
(ListIterator) tupleFilter.getChildren().listIterator();
+

[2/6] kylin git commit: KYLIN-2540 fix concat cascade issue

2017-04-10 Thread lidong

KYLIN-2540 fix concat cascade issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3296f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3296f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3296f6

Branch: refs/heads/yang22-hbase102
Commit: 1c3296f65a4939df57502e38f115f1516c27ef0e
Parents: 9d4d741
Author: lidongsjtu 
Authored: Mon Apr 10 21:50:32 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 21:54:11 2017 +0800

--
 build/conf/kylin.properties |  1 +
 .../test_case_data/sandbox/kylin.properties |  1 +
 .../src/test/resources/query/sql/query104.sql   | 31 
 .../org/apache/kylin/query/udf/ConcatUDF.java   | 28 ++
 .../org/apache/kylin/rest/util/QueryUtil.java   | 15 --
 5 files changed, 61 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 7b93c94..d00d714 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -149,6 +149,7 @@ kylin.snapshot.max-mb=300
 kylin.query.scan-threshold=1000
 
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 
 # 3G
 kylin.query.memory-budget-bytes=3221225472

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index a77a061..b4e32d6 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -104,6 +104,7 @@ kylin.storage.hbase.max-region-count=5
 kylin.storage.hbase.hfile-size-gb=2
 
 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
 
 # for test

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/kylin-it/src/test/resources/query/sql/query104.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query104.sql 
b/kylin-it/src/test/resources/query/sql/query104.sql
new file mode 100644
index 000..bb85664
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query104.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select concat(meta_categ_name, lstg_format_name) as c1, 
concat(meta_categ_name, 'CONST') as c2, concat(meta_categ_name, 
concat(test_sites.site_name, lstg_format_name)) as c3, count(1) as cnt, 
sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name 
not in ('Crafts','Computers'))
+ group by concat(meta_categ_name, lstg_format_name), concat(meta_categ_name, 
'CONST'), concat(meta_categ_name, concat(test_sites.site_name, 
lstg_format_name))
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java 
b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
new file mode 100644
index 000..9c0da53
--- /dev/null
+++

[6/6] kylin git commit: KYLIN-2233 Support HBase 1.0.2

2017-04-10 Thread lidong

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f71645e9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f71645e9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f71645e9

Branch: refs/heads/yang22-hbase102
Commit: f71645e94df26e7c1ea213d8872c969f46ae43bf
Parents: f245d67
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:28:56 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f71645e9/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dc23a2f..099316a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f71645e9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/f71645e9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of

[1/4] kylin git commit: KYLIN-2540 remove unused test [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 f2450cb6b -> c7678a9c3 (forced update)


KYLIN-2540 remove unused test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/35b7a291
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/35b7a291
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/35b7a291

Branch: refs/heads/yang22-cdh5.7
Commit: 35b7a29148222c425644c897d6129e123bbb01d7
Parents: 1c3296f
Author: lidongsjtu 
Authored: Mon Apr 10 22:22:19 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:22:19 2017 +0800

--
 .../test/java/org/apache/kylin/rest/util/QueryUtilTest.java| 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/35b7a291/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
--
diff --git 
a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java 
b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
index c00cd3f..dcd8c98 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
@@ -51,12 +51,6 @@ public class QueryUtilTest extends LocalFileMetadataTestCase 
{
 String s = QueryUtil.massageSql(sqlRequest);
 Assert.assertEquals("select ( date '2001-09-28' + interval '2' 
month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' 
month)", s);
 }
-{
-SQLRequest sqlRequest = new SQLRequest();
-sqlRequest.setSql("select 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 ()");
-String s = QueryUtil.massageSql(sqlRequest);
-Assert.assertEquals("select {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } ()", s);
-}
 }
 
 @Test

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f245d675
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f245d675
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f245d675

Branch: refs/heads/yang22-cdh5.7
Commit: f245d675a34c482a2ce05553c0173c394302e03f
Parents: 35b7a29
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:24:27 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2017-04-10 Thread lidong

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c7678a9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c7678a9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c7678a9c

Branch: refs/heads/yang22-cdh5.7
Commit: c7678a9c38ab3068af558eb155ffae3c3ad2c2b4
Parents: f245d67
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Mon Apr 10 22:26:47 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c7678a9c/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7678a9c/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7678a9c/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f245d675
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f245d675
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f245d675

Branch: refs/heads/yang22-hbase1.x
Commit: f245d675a34c482a2ce05553c0173c394302e03f
Parents: 35b7a29
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:24:27 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/f245d675/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[1/3] kylin git commit: KYLIN-2540 remove unused test [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x de4019465 -> f245d675a (forced update)


KYLIN-2540 remove unused test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/35b7a291
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/35b7a291
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/35b7a291

Branch: refs/heads/yang22-hbase1.x
Commit: 35b7a29148222c425644c897d6129e123bbb01d7
Parents: 1c3296f
Author: lidongsjtu 
Authored: Mon Apr 10 22:22:19 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:22:19 2017 +0800

--
 .../test/java/org/apache/kylin/rest/util/QueryUtilTest.java| 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/35b7a291/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
--
diff --git 
a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java 
b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
index c00cd3f..dcd8c98 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
@@ -51,12 +51,6 @@ public class QueryUtilTest extends LocalFileMetadataTestCase 
{
 String s = QueryUtil.massageSql(sqlRequest);
 Assert.assertEquals("select ( date '2001-09-28' + interval '2' 
month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' 
month)", s);
 }
-{
-SQLRequest sqlRequest = new SQLRequest();
-sqlRequest.setSql("select 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 ()");
-String s = QueryUtil.massageSql(sqlRequest);
-Assert.assertEquals("select {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } ()", s);
-}
 }
 
 @Test

kylin git commit: KYLIN-2540 remove unused test

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22 1c3296f65 -> 35b7a2914


KYLIN-2540 remove unused test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/35b7a291
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/35b7a291
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/35b7a291

Branch: refs/heads/yang22
Commit: 35b7a29148222c425644c897d6129e123bbb01d7
Parents: 1c3296f
Author: lidongsjtu 
Authored: Mon Apr 10 22:22:19 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:22:19 2017 +0800

--
 .../test/java/org/apache/kylin/rest/util/QueryUtilTest.java| 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/35b7a291/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
--
diff --git 
a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java 
b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
index c00cd3f..dcd8c98 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
@@ -51,12 +51,6 @@ public class QueryUtilTest extends LocalFileMetadataTestCase 
{
 String s = QueryUtil.massageSql(sqlRequest);
 Assert.assertEquals("select ( date '2001-09-28' + interval '2' 
month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' 
month)", s);
 }
-{
-SQLRequest sqlRequest = new SQLRequest();
-sqlRequest.setSql("select 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 ()");
-String s = QueryUtil.massageSql(sqlRequest);
-Assert.assertEquals("select {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } {fn 
concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\")
 } ()", s);
-}
 }
 
 @Test

[1/5] kylin git commit: KYLIN-2539 Useless filter dimension will impact cuboid selection. [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 fb823584a -> f2450cb6b (forced update)


KYLIN-2539 Useless filter dimension will impact cuboid selection.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d4d741b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d4d741b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d4d741b

Branch: refs/heads/yang22-cdh5.7
Commit: 9d4d741b254d08c9a5f30921478ebf956c09b9e9
Parents: ea778d2
Author: Hongbin Ma 
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 10 21:31:50 2017 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 14 +++
 .../filter/FilterOptimizeTransformer.java   | 89 
 .../gtrecord/GTCubeStorageQueryBase.java|  5 +-
 3 files changed, 107 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..f7cfb6a 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -215,6 +215,20 @@ public class CompareTupleFilter extends TupleFilter {
 return ((function != null && function.isEvaluable()) || column != 
null) && !conditionValues.isEmpty();
 }
 
+public boolean alwaysReturnTrue() {
+// 1 = 1
+if (this.operator == FilterOperatorEnum.EQ) {
+if (this.children != null && this.children.size() == 2 && //
+this.children.get(0) instanceof ConstantTupleFilter && //
+this.children.get(1) instanceof ConstantTupleFilter && //
+((ConstantTupleFilter) 
this.children.get(0)).getValues().equals(((ConstantTupleFilter) 
this.children.get(1)).getValues())) {
+return true;
+}
+}
+
+return false;
+}
+
 @SuppressWarnings({ "unchecked", "rawtypes" })
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 000..a3bbad6
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * 
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+public static final Logger logger = 
LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+@Override
+public TupleFilter transform(TupleFilter tupleFilter) {
+TupleFilter translated = null;
+if (tupleFilter instanceof CompareTupleFilter) {
+//normal case
+translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) 
tupleFilter);
+} else if (tupleFilter instanceof LogicalTupleFilter) {
+@SuppressWarnings("unchecked")
+ListIterator childIterator = 
(ListIterator) tupleFilter.getChildren().listIterator();
+while

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/de401946/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/de401946/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[2/5] kylin git commit: KYLIN-2540 fix concat cascade issue

2017-04-10 Thread lidong

KYLIN-2540 fix concat cascade issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3296f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3296f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3296f6

Branch: refs/heads/yang22-cdh5.7
Commit: 1c3296f65a4939df57502e38f115f1516c27ef0e
Parents: 9d4d741
Author: lidongsjtu 
Authored: Mon Apr 10 21:50:32 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 21:54:11 2017 +0800

--
 build/conf/kylin.properties |  1 +
 .../test_case_data/sandbox/kylin.properties |  1 +
 .../src/test/resources/query/sql/query104.sql   | 31 
 .../org/apache/kylin/query/udf/ConcatUDF.java   | 28 ++
 .../org/apache/kylin/rest/util/QueryUtil.java   | 15 --
 5 files changed, 61 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 7b93c94..d00d714 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -149,6 +149,7 @@ kylin.snapshot.max-mb=300
 kylin.query.scan-threshold=1000
 
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 
 # 3G
 kylin.query.memory-budget-bytes=3221225472

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index a77a061..b4e32d6 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -104,6 +104,7 @@ kylin.storage.hbase.max-region-count=5
 kylin.storage.hbase.hfile-size-gb=2
 
 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
 
 # for test

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/kylin-it/src/test/resources/query/sql/query104.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query104.sql 
b/kylin-it/src/test/resources/query/sql/query104.sql
new file mode 100644
index 000..bb85664
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query104.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select concat(meta_categ_name, lstg_format_name) as c1, 
concat(meta_categ_name, 'CONST') as c2, concat(meta_categ_name, 
concat(test_sites.site_name, lstg_format_name)) as c3, count(1) as cnt, 
sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name 
not in ('Crafts','Computers'))
+ group by concat(meta_categ_name, lstg_format_name), concat(meta_categ_name, 
'CONST'), concat(meta_categ_name, concat(test_sites.site_name, 
lstg_format_name))
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java 
b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
new file mode 100644
index 000..9c0da53
--- /dev/null
+++

[5/5] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2017-04-10 Thread lidong

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2450cb6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2450cb6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2450cb6

Branch: refs/heads/yang22-cdh5.7
Commit: f2450cb6b3d09f51c9c3215e5323a9b7d91e00cb
Parents: de40194
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Mon Apr 10 22:13:54 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/de401946/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/de401946/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[1/4] kylin git commit: KYLIN-2539 Useless filter dimension will impact cuboid selection. [Forced Update!]

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x a8709afc2 -> de4019465 (forced update)


KYLIN-2539 Useless filter dimension will impact cuboid selection.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d4d741b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d4d741b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d4d741b

Branch: refs/heads/yang22-hbase1.x
Commit: 9d4d741b254d08c9a5f30921478ebf956c09b9e9
Parents: ea778d2
Author: Hongbin Ma 
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 10 21:31:50 2017 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 14 +++
 .../filter/FilterOptimizeTransformer.java   | 89 
 .../gtrecord/GTCubeStorageQueryBase.java|  5 +-
 3 files changed, 107 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..f7cfb6a 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -215,6 +215,20 @@ public class CompareTupleFilter extends TupleFilter {
 return ((function != null && function.isEvaluable()) || column != 
null) && !conditionValues.isEmpty();
 }
 
+public boolean alwaysReturnTrue() {
+// 1 = 1
+if (this.operator == FilterOperatorEnum.EQ) {
+if (this.children != null && this.children.size() == 2 && //
+this.children.get(0) instanceof ConstantTupleFilter && //
+this.children.get(1) instanceof ConstantTupleFilter && //
+((ConstantTupleFilter) 
this.children.get(0)).getValues().equals(((ConstantTupleFilter) 
this.children.get(1)).getValues())) {
+return true;
+}
+}
+
+return false;
+}
+
 @SuppressWarnings({ "unchecked", "rawtypes" })
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 000..a3bbad6
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * 
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+public static final Logger logger = 
LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+@Override
+public TupleFilter transform(TupleFilter tupleFilter) {
+TupleFilter translated = null;
+if (tupleFilter instanceof CompareTupleFilter) {
+//normal case
+translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) 
tupleFilter);
+} else if (tupleFilter instanceof LogicalTupleFilter) {
+@SuppressWarnings("unchecked")
+ListIterator childIterator = 
(ListIterator) tupleFilter.getChildren().listIterator();
+

[2/4] kylin git commit: KYLIN-2540 fix concat cascade issue

2017-04-10 Thread lidong

KYLIN-2540 fix concat cascade issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3296f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3296f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3296f6

Branch: refs/heads/yang22-hbase1.x
Commit: 1c3296f65a4939df57502e38f115f1516c27ef0e
Parents: 9d4d741
Author: lidongsjtu 
Authored: Mon Apr 10 21:50:32 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 21:54:11 2017 +0800

--
 build/conf/kylin.properties |  1 +
 .../test_case_data/sandbox/kylin.properties |  1 +
 .../src/test/resources/query/sql/query104.sql   | 31 
 .../org/apache/kylin/query/udf/ConcatUDF.java   | 28 ++
 .../org/apache/kylin/rest/util/QueryUtil.java   | 15 --
 5 files changed, 61 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 7b93c94..d00d714 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -149,6 +149,7 @@ kylin.snapshot.max-mb=300
 kylin.query.scan-threshold=1000
 
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 
 # 3G
 kylin.query.memory-budget-bytes=3221225472

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index a77a061..b4e32d6 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -104,6 +104,7 @@ kylin.storage.hbase.max-region-count=5
 kylin.storage.hbase.hfile-size-gb=2
 
 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
 
 # for test

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/kylin-it/src/test/resources/query/sql/query104.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query104.sql 
b/kylin-it/src/test/resources/query/sql/query104.sql
new file mode 100644
index 000..bb85664
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query104.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select concat(meta_categ_name, lstg_format_name) as c1, 
concat(meta_categ_name, 'CONST') as c2, concat(meta_categ_name, 
concat(test_sites.site_name, lstg_format_name)) as c3, count(1) as cnt, 
sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name 
not in ('Crafts','Computers'))
+ group by concat(meta_categ_name, lstg_format_name), concat(meta_categ_name, 
'CONST'), concat(meta_categ_name, concat(test_sites.site_name, 
lstg_format_name))
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java 
b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
new file mode 100644
index 000..9c0da53
--- /dev/null
+++

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/de401946
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/de401946
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/de401946

Branch: refs/heads/yang22-hbase1.x
Commit: de40194655e6efba02f53a30a2662cb458bb3ce2
Parents: 1c3296f
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 22:12:03 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/de401946/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

kylin git commit: KYLIN-2540 fix concat cascade issue

2017-04-10 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22 9d4d741b2 -> 1c3296f65


KYLIN-2540 fix concat cascade issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3296f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3296f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3296f6

Branch: refs/heads/yang22
Commit: 1c3296f65a4939df57502e38f115f1516c27ef0e
Parents: 9d4d741
Author: lidongsjtu 
Authored: Mon Apr 10 21:50:32 2017 +0800
Committer: lidongsjtu 
Committed: Mon Apr 10 21:54:11 2017 +0800

--
 build/conf/kylin.properties |  1 +
 .../test_case_data/sandbox/kylin.properties |  1 +
 .../src/test/resources/query/sql/query104.sql   | 31 
 .../org/apache/kylin/query/udf/ConcatUDF.java   | 28 ++
 .../org/apache/kylin/rest/util/QueryUtil.java   | 15 --
 5 files changed, 61 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 7b93c94..d00d714 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -149,6 +149,7 @@ kylin.snapshot.max-mb=300
 kylin.query.scan-threshold=1000
 
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 
 # 3G
 kylin.query.memory-budget-bytes=3221225472

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index a77a061..b4e32d6 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -104,6 +104,7 @@ kylin.storage.hbase.max-region-count=5
 kylin.storage.hbase.hfile-size-gb=2
 
 kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
 
 # for test

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/kylin-it/src/test/resources/query/sql/query104.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query104.sql 
b/kylin-it/src/test/resources/query/sql/query104.sql
new file mode 100644
index 000..bb85664
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query104.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select concat(meta_categ_name, lstg_format_name) as c1, 
concat(meta_categ_name, 'CONST') as c2, concat(meta_categ_name, 
concat(test_sites.site_name, lstg_format_name)) as c3, count(1) as cnt, 
sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name 
not in ('Crafts','Computers'))
+ group by concat(meta_categ_name, lstg_format_name), concat(meta_categ_name, 
'CONST'), concat(meta_categ_name, concat(test_sites.site_name, 
lstg_format_name))
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3296f6/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java 
b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
new file mode 100644

kylin git commit: KYLIN-2539 Useless filter dimension will impact cuboid selection.

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang22 ea778d2c9 -> 9d4d741b2


KYLIN-2539 Useless filter dimension will impact cuboid selection.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d4d741b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d4d741b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d4d741b

Branch: refs/heads/yang22
Commit: 9d4d741b254d08c9a5f30921478ebf956c09b9e9
Parents: ea778d2
Author: Hongbin Ma 
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 10 21:31:50 2017 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 14 +++
 .../filter/FilterOptimizeTransformer.java   | 89 
 .../gtrecord/GTCubeStorageQueryBase.java|  5 +-
 3 files changed, 107 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..f7cfb6a 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -215,6 +215,20 @@ public class CompareTupleFilter extends TupleFilter {
 return ((function != null && function.isEvaluable()) || column != 
null) && !conditionValues.isEmpty();
 }
 
+public boolean alwaysReturnTrue() {
+// 1 = 1
+if (this.operator == FilterOperatorEnum.EQ) {
+if (this.children != null && this.children.size() == 2 && //
+this.children.get(0) instanceof ConstantTupleFilter && //
+this.children.get(1) instanceof ConstantTupleFilter && //
+((ConstantTupleFilter) 
this.children.get(0)).getValues().equals(((ConstantTupleFilter) 
this.children.get(1)).getValues())) {
+return true;
+}
+}
+
+return false;
+}
+
 @SuppressWarnings({ "unchecked", "rawtypes" })
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9d4d741b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 000..a3bbad6
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * 
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+public static final Logger logger = 
LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+@Override
+public TupleFilter transform(TupleFilter tupleFilter) {
+TupleFilter translated = null;
+if (tupleFilter instanceof CompareTupleFilter) {
+//normal case
+translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) 
tupleFilter);
+} else if (tupleFilter instanceof LogicalTupleFilter) {
+@SuppressWarnings("unchecked")
+ListIterator childIterator = 
(ListIterator) tupleFilter.getChildren().listIterator();
+while (childIterator.hasNext()) {
+

kylin git commit: KYLIN-2539 Useless filter dimension will impact cuboid selection.

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/2.0.x 41e383c49 -> 273a40989


KYLIN-2539 Useless filter dimension will impact cuboid selection.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/273a4098
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/273a4098
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/273a4098

Branch: refs/heads/2.0.x
Commit: 273a409899e4a392e7ddf8f36eaabe3b46d5fdba
Parents: 41e383c
Author: Hongbin Ma 
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 10 21:29:23 2017 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 14 +++
 .../filter/FilterOptimizeTransformer.java   | 89 
 .../gtrecord/GTCubeStorageQueryBase.java|  6 +-
 3 files changed, 108 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/273a4098/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d567c89..d783e52 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -218,6 +218,20 @@ public class CompareTupleFilter extends TupleFilter {
 && !conditionValues.isEmpty() && secondColumn == null;
 }
 
+public boolean alwaysReturnTrue() {
+// 1 = 1
+if (this.operator == FilterOperatorEnum.EQ) {
+if (this.children != null && this.children.size() == 2 && //
+this.children.get(0) instanceof ConstantTupleFilter && //
+this.children.get(1) instanceof ConstantTupleFilter && //
+((ConstantTupleFilter) 
this.children.get(0)).getValues().equals(((ConstantTupleFilter) 
this.children.get(1)).getValues())) {
+return true;
+}
+}
+
+return false;
+}
+
 @SuppressWarnings({ "unchecked", "rawtypes" })
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/273a4098/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 000..a3bbad6
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * 
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+public static final Logger logger = 
LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+@Override
+public TupleFilter transform(TupleFilter tupleFilter) {
+TupleFilter translated = null;
+if (tupleFilter instanceof CompareTupleFilter) {
+//normal case
+translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) 
tupleFilter);
+} else if (tupleFilter instanceof LogicalTupleFilter) {
+@SuppressWarnings("unchecked")
+ListIterator childIterator = 
(ListIterator) tupleFilter.getChildren().listIterator();
+while (childIterator.hasNext()) {
+TupleFilter transformed =

kylin git commit: KYLIN-2539 Useless filter dimension will impact cuboid selection.

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master c17b3eb36 -> 177079d10


KYLIN-2539 Useless filter dimension will impact cuboid selection.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/177079d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/177079d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/177079d1

Branch: refs/heads/master
Commit: 177079d10c1dfd4decadd743eafdfc36a712e470
Parents: c17b3eb
Author: Hongbin Ma 
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 10 21:22:09 2017 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 14 +++
 .../filter/FilterOptimizeTransformer.java   | 89 
 .../gtrecord/GTCubeStorageQueryBase.java|  6 +-
 3 files changed, 108 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d567c89..d783e52 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -218,6 +218,20 @@ public class CompareTupleFilter extends TupleFilter {
 && !conditionValues.isEmpty() && secondColumn == null;
 }
 
+public boolean alwaysReturnTrue() {
+// 1 = 1
+if (this.operator == FilterOperatorEnum.EQ) {
+if (this.children != null && this.children.size() == 2 && //
+this.children.get(0) instanceof ConstantTupleFilter && //
+this.children.get(1) instanceof ConstantTupleFilter && //
+((ConstantTupleFilter) 
this.children.get(0)).getValues().equals(((ConstantTupleFilter) 
this.children.get(1)).getValues())) {
+return true;
+}
+}
+
+return false;
+}
+
 @SuppressWarnings({ "unchecked", "rawtypes" })
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 000..a3bbad6
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * 
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+public static final Logger logger = 
LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+@Override
+public TupleFilter transform(TupleFilter tupleFilter) {
+TupleFilter translated = null;
+if (tupleFilter instanceof CompareTupleFilter) {
+//normal case
+translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) 
tupleFilter);
+} else if (tupleFilter instanceof LogicalTupleFilter) {
+@SuppressWarnings("unchecked")
+ListIterator childIterator = 
(ListIterator) tupleFilter.getChildren().listIterator();
+while (childIterator.hasNext()) {
+TupleFilter transformed =

[5/9] kylin git commit: KYLIN-2537 hbase r/w bug fix

2017-04-10 Thread billyliu

KYLIN-2537 hbase r/w bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c82a0d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c82a0d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c82a0d0

Branch: refs/heads/yang22-hbase102
Commit: 9c82a0d0ec138efc04c5393ce475910b887ad78f
Parents: 09e7545
Author: Li Yang 
Authored: Mon Apr 10 14:12:18 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:21:20 2017 +0800

--
 .../java/org/apache/kylin/storage/hbase/HBaseConnection.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c82a0d0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 335bfe7..d7311a9 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -202,10 +202,13 @@ public class HBaseConnection {
 conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, 
nameServices.toArray(new String[0]));
 }
 
-public static String makeQualifiedPathInHBaseCluster(String path) {
+public static String makeQualifiedPathInHBaseCluster(String inPath) {
+Path path = new Path(inPath);
+path = Path.getPathWithoutSchemeAndAuthority(path);
+
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-return fs.makeQualified(new Path(path)).toString();
+return fs.makeQualified(path).toString();
 } catch (IOException e) {
 throw new IllegalArgumentException("Cannot create FileSystem from 
current hbase cluster conf", e);
 }

[3/9] kylin git commit: KYLIN-2528 refine job email notification to support starttls and customized port

2017-04-10 Thread billyliu

KYLIN-2528 refine job email notification to support starttls and customized port


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6617955b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6617955b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6617955b

Branch: refs/heads/yang22-hbase102
Commit: 6617955b4fe065af9da46b2a9bc665a466e0
Parents: 0c6441c
Author: Hongbin Ma 
Authored: Fri Mar 31 21:15:00 2017 +0800
Committer: Hongbin Ma 
Committed: Fri Mar 31 21:15:00 2017 +0800

--
 build/conf/kylin.properties  | 13 -
 .../org/apache/kylin/common/KylinConfigBase.java |  9 +
 .../org/apache/kylin/common/util/MailService.java| 15 +--
 examples/test_case_data/localmeta/kylin.properties   | 15 +--
 examples/test_case_data/sandbox/kylin.properties |  9 -
 5 files changed, 47 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 5e7d235..7b93c94 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -109,11 +109,14 @@ kylin.job.status.with.kerberos=false
 kylin.job.step.timeout=7200
 
 # If true, will send email notification on job complete
-kylin.job.notification-enabled=false
-kylin.job.notification-mail-host=
-kylin.job.notification-mail-username=
-kylin.job.notification-mail-password=
-kylin.job.notification-mail-sender=
+#kylin.job.notification-enabled=true
+#kylin.job.notification-mail-enable-starttls=true
+#kylin.job.notification-mail-host=smtp.office365.com
+#kylin.job.notification-mail-port=587
+#kylin.job.notification-mail-username=ky...@example.com
+#kylin.job.notification-mail-password=mypassword
+#kylin.job.notification-mail-sender=ky...@example.com
+
 
 
 ### ENGINE ###

http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5b7c734..ac9ed04 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -402,6 +402,15 @@ abstract public class KylinConfigBase implements 
Serializable {
 setProperty("kylin.job.notification-enabled", "" + enable);
 }
 
+public boolean isStarttlsEnabled() {
+return 
Boolean.parseBoolean(getOptional("kylin.job.notification-mail-enable-starttls", 
"false"));
+}
+
+public String getSmtpPort() {
+return getOptional("kylin.job.notification-mail-port", "25");
+}
+
+
 public String getMailHost() {
 return getOptional("kylin.job.notification-mail-host", "");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java 
b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
index 5793967..3bccd8f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
@@ -34,18 +34,22 @@ public class MailService {
 private static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(MailService.class);
 
 private Boolean enabled = Boolean.TRUE;
+private Boolean starttlsEnabled = Boolean.FALSE;
 private String host;
+private String port;
 private String username;
 private String password;
 private String sender;
 
 public MailService(KylinConfig config) {
-this(config.isMailEnabled(), config.getMailHost(), 
config.getMailUsername(), config.getMailPassword(), config.getMailSender());
+this(config.isMailEnabled(), config.isStarttlsEnabled(), 
config.getMailHost(), config.getSmtpPort(), config.getMailUsername(), 
config.getMailPassword(), config.getMailSender());
 }
 
-private MailService(boolean enabled, String host, String username, String 
password, String sender) {
+private MailService(boolean enabled, boolean starttlsEnabled, String host, 
String port, String username, String password, String sender) {
 this.enabled = enabled;
+this.starttlsEnabled = starttlsEnabled;
 this.host = host;
+this.port =

[8/9] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8709afc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8709afc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8709afc

Branch: refs/heads/yang22-hbase102
Commit: a8709afc2b40718897c4e4f1d0a9b9f71227eeb0
Parents: ea778d2
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Mon Apr 10 18:39:00 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[7/9] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[1/9] kylin git commit: KYLIN-2497 fix minor bug to support long type [Forced Update!]

2017-04-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 0ac93144d -> a055a4cd9 (forced update)


KYLIN-2497 fix minor bug to support long type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50955963
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50955963
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50955963

Branch: refs/heads/yang22-hbase102
Commit: 50955963d117a8a0a05b756cecbb6b1c6d06647d
Parents: e477b51
Author: Hongbin Ma 
Authored: Sat Mar 11 16:58:06 2017 +0800
Committer: Hongbin Ma 
Committed: Sat Mar 11 16:58:06 2017 +0800

--
 .../org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/50955963/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index acec972..bb4152e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -50,7 +50,7 @@ public class SelfDefineSortableKey implements 
WritableComparable

[2/9] kylin git commit: stablize BuildCubeWithStream

2017-04-10 Thread billyliu

stablize BuildCubeWithStream


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c6441c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c6441c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c6441c7

Branch: refs/heads/yang22-hbase102
Commit: 0c6441c710ddefaee6285d41f4eb20e06533c9f8
Parents: 5095596
Author: shaofengshi 
Authored: Wed Mar 15 12:02:52 2017 +0800
Committer: shaofengshi 
Committed: Wed Mar 15 12:02:52 2017 +0800

--
 .../kylin/provision/BuildCubeWithStream.java| 76 
 .../kylin/storage/hbase/util/ZookeeperUtil.java | 52 ++
 2 files changed, 115 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c6441c7/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 8abb84c..030b7d6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -36,6 +36,10 @@ import java.util.concurrent.TimeUnit;
 
 import org.I0Itec.zkclient.ZkConnection;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
@@ -63,6 +67,7 @@ import org.apache.kylin.source.kafka.KafkaConfigManager;
 import org.apache.kylin.source.kafka.config.BrokerConfig;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
+import org.apache.kylin.storage.hbase.util.ZookeeperUtil;
 import org.apache.kylin.tool.StorageCleanupJob;
 import org.junit.Assert;
 import org.slf4j.Logger;
@@ -84,8 +89,14 @@ public class BuildCubeWithStream {
 
 private KafkaConfig kafkaConfig;
 private MockKafka kafkaServer;
+private ZkConnection zkConnection;
+private final String kafkaZkPath = "/" + UUID.randomUUID().toString();
+
 protected static boolean fastBuildMode = false;
-private boolean generateData = true;
+private volatile boolean generateData = true;
+private volatile boolean generateDataDone = false;
+
+private static final int BUILD_ROUND = 5;
 
 public void before() throws Exception {
 deployEnv();
@@ -126,8 +137,9 @@ public class BuildCubeWithStream {
 
 private void startEmbeddedKafka(String topicName, BrokerConfig 
brokerConfig) {
 //Start mock Kakfa
-String zkConnectionStr = "sandbox:2181";
-ZkConnection zkConnection = new ZkConnection(zkConnectionStr);
+String zkConnectionStr = ZookeeperUtil.getZKConnectString() + 
kafkaZkPath;
+System.out.println("zkConnectionStr" + zkConnectionStr);
+zkConnection = new ZkConnection(zkConnectionStr);
 // Assert.assertEquals(ZooKeeper.States.CONNECTED, 
zkConnection.getZookeeperState());
 kafkaServer = new MockKafka(zkConnection, brokerConfig.getPort(), 
brokerConfig.getId());
 kafkaServer.start();
@@ -171,18 +183,33 @@ public class BuildCubeWithStream {
 try {
 generateStreamData(dateStart, dateEnd, 
rand.nextInt(100));
 dateStart = dateEnd;
-sleep(rand.nextInt(rand.nextInt(100 * 1000))); // wait 
random time
+sleep(rand.nextInt(rand.nextInt(30)) * 1000); // wait 
random time
 } catch (Exception e) {
 e.printStackTrace();
 }
 }
+generateDataDone = true;
 }
 }).start();
 ExecutorService executorService = Executors.newCachedThreadPool();
 
 List futures = Lists.newArrayList();
-for (int i = 0; i < 5; i++) {
-Thread.sleep(2 * 60 * 1000); // wait for new messages
+for (int i = 0; i < BUILD_ROUND; i++) {
+if (i == (BUILD_ROUND - 1)) {
+// stop generating message to kafka
+generateData = false;
+int waittime = 0;
+while (generateDataDone == false && waittime < 100) {
+Thread.sleep(1000);
+waittime++;
+}
+if (generateDataDone == false) {
+

[9/9] kylin git commit: KYLIN-2233 Support HBase 1.0.2

2017-04-10 Thread billyliu

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a055a4cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a055a4cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a055a4cd

Branch: refs/heads/yang22-hbase102
Commit: a055a4cd97ae24756613870d429c96add9dcdd44
Parents: a8709af
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Billy Liu 
Committed: Mon Apr 10 18:42:56 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a055a4cd/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dc23a2f..099316a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a055a4cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/a055a4cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of

[6/9] kylin git commit: KYLIN-2537 fix DeployCoprocessorCLI

2017-04-10 Thread billyliu

KYLIN-2537 fix DeployCoprocessorCLI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea778d2c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea778d2c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea778d2c

Branch: refs/heads/yang22-hbase102
Commit: ea778d2c9748acfe9931bcdd91972cba48630f9c
Parents: 9c82a0d
Author: Li Yang 
Authored: Mon Apr 10 18:07:50 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:21:24 2017 +0800

--
 .../org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea778d2c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..8d20792 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -382,6 +382,7 @@ public class DeployCoprocessorCLI {
 
 private static Path getCoprocessorHDFSDir(FileSystem fileSystem, 
KylinConfig config) throws IOException {
 String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
+hdfsWorkingDirectory = 
HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
 Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
 fileSystem.mkdirs(coprocessorDir);
 return coprocessorDir;

[4/9] kylin git commit: support azure hdinsight counters

2017-04-10 Thread billyliu

support azure hdinsight counters


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09e75450
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09e75450
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09e75450

Branch: refs/heads/yang22-hbase102
Commit: 09e75450879281b5921ffa24cd5fc6edc554ee73
Parents: 6617955
Author: shaofengshi 
Authored: Wed Mar 8 11:23:29 2017 +0800
Committer: shaofengshi 
Committed: Sat Apr 1 08:56:09 2017 +0800

--
 .../kylin/engine/mr/common/HadoopCmdOutput.java   | 14 --
 1 file changed, 12 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/09e75450/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 9d016cc..023717f 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -22,6 +22,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskCounter;
@@ -93,9 +94,18 @@ public class HadoopCmdOutput {
 this.output.append(counters.toString()).append("\n");
 logger.debug(counters.toString());
 
+String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN";
+String bytsReadCounterName = "HDFS_BYTES_READ";
+String fsScheme = 
FileSystem.get(job.getConfiguration()).getScheme();
+if (("wasb").equalsIgnoreCase(fsScheme)) {
+// for Azure blob store
+bytsWrittenCounterName = "WASB_BYTES_WRITTEN";
+bytsReadCounterName = "WASB_BYTES_READ";
+}
+
 mapInputRecords = 
String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
-hdfsBytesWritten = 
String.valueOf(counters.findCounter("FileSystemCounters", 
"HDFS_BYTES_WRITTEN").getValue());
-hdfsBytesRead = 
String.valueOf(counters.findCounter("FileSystemCounters", 
"HDFS_BYTES_READ").getValue());
+hdfsBytesWritten = 
String.valueOf(counters.findCounter("FileSystemCounters", 
bytsWrittenCounterName).getValue());
+hdfsBytesRead = 
String.valueOf(counters.findCounter("FileSystemCounters", 
bytsReadCounterName).getValue());
 } catch (Exception e) {
 logger.error(e.getLocalizedMessage(), e);
 output.append(e.getLocalizedMessage());

[2/9] kylin git commit: stablize BuildCubeWithStream

2017-04-10 Thread billyliu

stablize BuildCubeWithStream


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c6441c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c6441c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c6441c7

Branch: refs/heads/yang22-cdh5.7
Commit: 0c6441c710ddefaee6285d41f4eb20e06533c9f8
Parents: 5095596
Author: shaofengshi 
Authored: Wed Mar 15 12:02:52 2017 +0800
Committer: shaofengshi 
Committed: Wed Mar 15 12:02:52 2017 +0800

--
 .../kylin/provision/BuildCubeWithStream.java| 76 
 .../kylin/storage/hbase/util/ZookeeperUtil.java | 52 ++
 2 files changed, 115 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c6441c7/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 8abb84c..030b7d6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -36,6 +36,10 @@ import java.util.concurrent.TimeUnit;
 
 import org.I0Itec.zkclient.ZkConnection;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
@@ -63,6 +67,7 @@ import org.apache.kylin.source.kafka.KafkaConfigManager;
 import org.apache.kylin.source.kafka.config.BrokerConfig;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
+import org.apache.kylin.storage.hbase.util.ZookeeperUtil;
 import org.apache.kylin.tool.StorageCleanupJob;
 import org.junit.Assert;
 import org.slf4j.Logger;
@@ -84,8 +89,14 @@ public class BuildCubeWithStream {
 
 private KafkaConfig kafkaConfig;
 private MockKafka kafkaServer;
+private ZkConnection zkConnection;
+private final String kafkaZkPath = "/" + UUID.randomUUID().toString();
+
 protected static boolean fastBuildMode = false;
-private boolean generateData = true;
+private volatile boolean generateData = true;
+private volatile boolean generateDataDone = false;
+
+private static final int BUILD_ROUND = 5;
 
 public void before() throws Exception {
 deployEnv();
@@ -126,8 +137,9 @@ public class BuildCubeWithStream {
 
 private void startEmbeddedKafka(String topicName, BrokerConfig 
brokerConfig) {
 //Start mock Kakfa
-String zkConnectionStr = "sandbox:2181";
-ZkConnection zkConnection = new ZkConnection(zkConnectionStr);
+String zkConnectionStr = ZookeeperUtil.getZKConnectString() + 
kafkaZkPath;
+System.out.println("zkConnectionStr" + zkConnectionStr);
+zkConnection = new ZkConnection(zkConnectionStr);
 // Assert.assertEquals(ZooKeeper.States.CONNECTED, 
zkConnection.getZookeeperState());
 kafkaServer = new MockKafka(zkConnection, brokerConfig.getPort(), 
brokerConfig.getId());
 kafkaServer.start();
@@ -171,18 +183,33 @@ public class BuildCubeWithStream {
 try {
 generateStreamData(dateStart, dateEnd, 
rand.nextInt(100));
 dateStart = dateEnd;
-sleep(rand.nextInt(rand.nextInt(100 * 1000))); // wait 
random time
+sleep(rand.nextInt(rand.nextInt(30)) * 1000); // wait 
random time
 } catch (Exception e) {
 e.printStackTrace();
 }
 }
+generateDataDone = true;
 }
 }).start();
 ExecutorService executorService = Executors.newCachedThreadPool();
 
 List futures = Lists.newArrayList();
-for (int i = 0; i < 5; i++) {
-Thread.sleep(2 * 60 * 1000); // wait for new messages
+for (int i = 0; i < BUILD_ROUND; i++) {
+if (i == (BUILD_ROUND - 1)) {
+// stop generating message to kafka
+generateData = false;
+int waittime = 0;
+while (generateDataDone == false && waittime < 100) {
+Thread.sleep(1000);
+waittime++;
+}
+if (generateDataDone == false) {
+

[6/9] kylin git commit: KYLIN-2537 fix DeployCoprocessorCLI

2017-04-10 Thread billyliu

KYLIN-2537 fix DeployCoprocessorCLI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea778d2c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea778d2c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea778d2c

Branch: refs/heads/yang22-cdh5.7
Commit: ea778d2c9748acfe9931bcdd91972cba48630f9c
Parents: 9c82a0d
Author: Li Yang 
Authored: Mon Apr 10 18:07:50 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:21:24 2017 +0800

--
 .../org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea778d2c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..8d20792 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -382,6 +382,7 @@ public class DeployCoprocessorCLI {
 
 private static Path getCoprocessorHDFSDir(FileSystem fileSystem, 
KylinConfig config) throws IOException {
 String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
+hdfsWorkingDirectory = 
HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
 Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
 fileSystem.mkdirs(coprocessorDir);
 return coprocessorDir;

[4/9] kylin git commit: support azure hdinsight counters

2017-04-10 Thread billyliu

support azure hdinsight counters


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09e75450
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09e75450
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09e75450

Branch: refs/heads/yang22-cdh5.7
Commit: 09e75450879281b5921ffa24cd5fc6edc554ee73
Parents: 6617955
Author: shaofengshi 
Authored: Wed Mar 8 11:23:29 2017 +0800
Committer: shaofengshi 
Committed: Sat Apr 1 08:56:09 2017 +0800

--
 .../kylin/engine/mr/common/HadoopCmdOutput.java   | 14 --
 1 file changed, 12 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/09e75450/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 9d016cc..023717f 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -22,6 +22,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskCounter;
@@ -93,9 +94,18 @@ public class HadoopCmdOutput {
 this.output.append(counters.toString()).append("\n");
 logger.debug(counters.toString());
 
+String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN";
+String bytsReadCounterName = "HDFS_BYTES_READ";
+String fsScheme = 
FileSystem.get(job.getConfiguration()).getScheme();
+if (("wasb").equalsIgnoreCase(fsScheme)) {
+// for Azure blob store
+bytsWrittenCounterName = "WASB_BYTES_WRITTEN";
+bytsReadCounterName = "WASB_BYTES_READ";
+}
+
 mapInputRecords = 
String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
-hdfsBytesWritten = 
String.valueOf(counters.findCounter("FileSystemCounters", 
"HDFS_BYTES_WRITTEN").getValue());
-hdfsBytesRead = 
String.valueOf(counters.findCounter("FileSystemCounters", 
"HDFS_BYTES_READ").getValue());
+hdfsBytesWritten = 
String.valueOf(counters.findCounter("FileSystemCounters", 
bytsWrittenCounterName).getValue());
+hdfsBytesRead = 
String.valueOf(counters.findCounter("FileSystemCounters", 
bytsReadCounterName).getValue());
 } catch (Exception e) {
 logger.error(e.getLocalizedMessage(), e);
 output.append(e.getLocalizedMessage());

[8/9] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8709afc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8709afc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8709afc

Branch: refs/heads/yang22-cdh5.7
Commit: a8709afc2b40718897c4e4f1d0a9b9f71227eeb0
Parents: ea778d2
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Mon Apr 10 18:39:00 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[9/9] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2017-04-10 Thread billyliu

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb823584
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb823584
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb823584

Branch: refs/heads/yang22-cdh5.7
Commit: fb823584a9a2e55d290d848e260445c2419af02a
Parents: a8709af
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Mon Apr 10 18:40:59 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fb823584/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb823584/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fb823584/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[7/9] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[3/9] kylin git commit: KYLIN-2528 refine job email notification to support starttls and customized port

2017-04-10 Thread billyliu

KYLIN-2528 refine job email notification to support starttls and customized port


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6617955b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6617955b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6617955b

Branch: refs/heads/yang22-cdh5.7
Commit: 6617955b4fe065af9da46b2a9bc665a466e0
Parents: 0c6441c
Author: Hongbin Ma 
Authored: Fri Mar 31 21:15:00 2017 +0800
Committer: Hongbin Ma 
Committed: Fri Mar 31 21:15:00 2017 +0800

--
 build/conf/kylin.properties  | 13 -
 .../org/apache/kylin/common/KylinConfigBase.java |  9 +
 .../org/apache/kylin/common/util/MailService.java| 15 +--
 examples/test_case_data/localmeta/kylin.properties   | 15 +--
 examples/test_case_data/sandbox/kylin.properties |  9 -
 5 files changed, 47 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 5e7d235..7b93c94 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -109,11 +109,14 @@ kylin.job.status.with.kerberos=false
 kylin.job.step.timeout=7200
 
 # If true, will send email notification on job complete
-kylin.job.notification-enabled=false
-kylin.job.notification-mail-host=
-kylin.job.notification-mail-username=
-kylin.job.notification-mail-password=
-kylin.job.notification-mail-sender=
+#kylin.job.notification-enabled=true
+#kylin.job.notification-mail-enable-starttls=true
+#kylin.job.notification-mail-host=smtp.office365.com
+#kylin.job.notification-mail-port=587
+#kylin.job.notification-mail-username=ky...@example.com
+#kylin.job.notification-mail-password=mypassword
+#kylin.job.notification-mail-sender=ky...@example.com
+
 
 
 ### ENGINE ###

http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5b7c734..ac9ed04 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -402,6 +402,15 @@ abstract public class KylinConfigBase implements 
Serializable {
 setProperty("kylin.job.notification-enabled", "" + enable);
 }
 
+public boolean isStarttlsEnabled() {
+return 
Boolean.parseBoolean(getOptional("kylin.job.notification-mail-enable-starttls", 
"false"));
+}
+
+public String getSmtpPort() {
+return getOptional("kylin.job.notification-mail-port", "25");
+}
+
+
 public String getMailHost() {
 return getOptional("kylin.job.notification-mail-host", "");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java 
b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
index 5793967..3bccd8f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
@@ -34,18 +34,22 @@ public class MailService {
 private static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(MailService.class);
 
 private Boolean enabled = Boolean.TRUE;
+private Boolean starttlsEnabled = Boolean.FALSE;
 private String host;
+private String port;
 private String username;
 private String password;
 private String sender;
 
 public MailService(KylinConfig config) {
-this(config.isMailEnabled(), config.getMailHost(), 
config.getMailUsername(), config.getMailPassword(), config.getMailSender());
+this(config.isMailEnabled(), config.isStarttlsEnabled(), 
config.getMailHost(), config.getSmtpPort(), config.getMailUsername(), 
config.getMailPassword(), config.getMailSender());
 }
 
-private MailService(boolean enabled, String host, String username, String 
password, String sender) {
+private MailService(boolean enabled, boolean starttlsEnabled, String host, 
String port, String username, String password, String sender) {
 this.enabled = enabled;
+this.starttlsEnabled = starttlsEnabled;
 this.host = host;
+this.port =

[5/9] kylin git commit: KYLIN-2537 hbase r/w bug fix

2017-04-10 Thread billyliu

KYLIN-2537 hbase r/w bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c82a0d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c82a0d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c82a0d0

Branch: refs/heads/yang22-cdh5.7
Commit: 9c82a0d0ec138efc04c5393ce475910b887ad78f
Parents: 09e7545
Author: Li Yang 
Authored: Mon Apr 10 14:12:18 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:21:20 2017 +0800

--
 .../java/org/apache/kylin/storage/hbase/HBaseConnection.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c82a0d0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 335bfe7..d7311a9 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -202,10 +202,13 @@ public class HBaseConnection {
 conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, 
nameServices.toArray(new String[0]));
 }
 
-public static String makeQualifiedPathInHBaseCluster(String path) {
+public static String makeQualifiedPathInHBaseCluster(String inPath) {
+Path path = new Path(inPath);
+path = Path.getPathWithoutSchemeAndAuthority(path);
+
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-return fs.makeQualified(new Path(path)).toString();
+return fs.makeQualified(path).toString();
 } catch (IOException e) {
 throw new IllegalArgumentException("Cannot create FileSystem from 
current hbase cluster conf", e);
 }

[1/9] kylin git commit: KYLIN-2497 fix minor bug to support long type [Forced Update!]

2017-04-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 760de8238 -> fb823584a (forced update)


KYLIN-2497 fix minor bug to support long type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50955963
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50955963
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50955963

Branch: refs/heads/yang22-cdh5.7
Commit: 50955963d117a8a0a05b756cecbb6b1c6d06647d
Parents: e477b51
Author: Hongbin Ma 
Authored: Sat Mar 11 16:58:06 2017 +0800
Committer: Hongbin Ma 
Committed: Sat Mar 11 16:58:06 2017 +0800

--
 .../org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/50955963/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index acec972..bb4152e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -50,7 +50,7 @@ public class SelfDefineSortableKey implements 
WritableComparable

[2/8] kylin git commit: stablize BuildCubeWithStream

2017-04-10 Thread billyliu

stablize BuildCubeWithStream


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c6441c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c6441c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c6441c7

Branch: refs/heads/yang22-hbase1.x
Commit: 0c6441c710ddefaee6285d41f4eb20e06533c9f8
Parents: 5095596
Author: shaofengshi 
Authored: Wed Mar 15 12:02:52 2017 +0800
Committer: shaofengshi 
Committed: Wed Mar 15 12:02:52 2017 +0800

--
 .../kylin/provision/BuildCubeWithStream.java| 76 
 .../kylin/storage/hbase/util/ZookeeperUtil.java | 52 ++
 2 files changed, 115 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c6441c7/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 8abb84c..030b7d6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -36,6 +36,10 @@ import java.util.concurrent.TimeUnit;
 
 import org.I0Itec.zkclient.ZkConnection;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
@@ -63,6 +67,7 @@ import org.apache.kylin.source.kafka.KafkaConfigManager;
 import org.apache.kylin.source.kafka.config.BrokerConfig;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
+import org.apache.kylin.storage.hbase.util.ZookeeperUtil;
 import org.apache.kylin.tool.StorageCleanupJob;
 import org.junit.Assert;
 import org.slf4j.Logger;
@@ -84,8 +89,14 @@ public class BuildCubeWithStream {
 
 private KafkaConfig kafkaConfig;
 private MockKafka kafkaServer;
+private ZkConnection zkConnection;
+private final String kafkaZkPath = "/" + UUID.randomUUID().toString();
+
 protected static boolean fastBuildMode = false;
-private boolean generateData = true;
+private volatile boolean generateData = true;
+private volatile boolean generateDataDone = false;
+
+private static final int BUILD_ROUND = 5;
 
 public void before() throws Exception {
 deployEnv();
@@ -126,8 +137,9 @@ public class BuildCubeWithStream {
 
 private void startEmbeddedKafka(String topicName, BrokerConfig 
brokerConfig) {
 //Start mock Kakfa
-String zkConnectionStr = "sandbox:2181";
-ZkConnection zkConnection = new ZkConnection(zkConnectionStr);
+String zkConnectionStr = ZookeeperUtil.getZKConnectString() + 
kafkaZkPath;
+System.out.println("zkConnectionStr" + zkConnectionStr);
+zkConnection = new ZkConnection(zkConnectionStr);
 // Assert.assertEquals(ZooKeeper.States.CONNECTED, 
zkConnection.getZookeeperState());
 kafkaServer = new MockKafka(zkConnection, brokerConfig.getPort(), 
brokerConfig.getId());
 kafkaServer.start();
@@ -171,18 +183,33 @@ public class BuildCubeWithStream {
 try {
 generateStreamData(dateStart, dateEnd, 
rand.nextInt(100));
 dateStart = dateEnd;
-sleep(rand.nextInt(rand.nextInt(100 * 1000))); // wait 
random time
+sleep(rand.nextInt(rand.nextInt(30)) * 1000); // wait 
random time
 } catch (Exception e) {
 e.printStackTrace();
 }
 }
+generateDataDone = true;
 }
 }).start();
 ExecutorService executorService = Executors.newCachedThreadPool();
 
 List futures = Lists.newArrayList();
-for (int i = 0; i < 5; i++) {
-Thread.sleep(2 * 60 * 1000); // wait for new messages
+for (int i = 0; i < BUILD_ROUND; i++) {
+if (i == (BUILD_ROUND - 1)) {
+// stop generating message to kafka
+generateData = false;
+int waittime = 0;
+while (generateDataDone == false && waittime < 100) {
+Thread.sleep(1000);
+waittime++;
+}
+if (generateDataDone == false) {
+

[6/8] kylin git commit: KYLIN-2537 fix DeployCoprocessorCLI

2017-04-10 Thread billyliu

KYLIN-2537 fix DeployCoprocessorCLI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea778d2c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea778d2c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea778d2c

Branch: refs/heads/yang22-hbase1.x
Commit: ea778d2c9748acfe9931bcdd91972cba48630f9c
Parents: 9c82a0d
Author: Li Yang 
Authored: Mon Apr 10 18:07:50 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:21:24 2017 +0800

--
 .../org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea778d2c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..8d20792 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -382,6 +382,7 @@ public class DeployCoprocessorCLI {
 
 private static Path getCoprocessorHDFSDir(FileSystem fileSystem, 
KylinConfig config) throws IOException {
 String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
+hdfsWorkingDirectory = 
HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
 Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
 fileSystem.mkdirs(coprocessorDir);
 return coprocessorDir;

[8/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8709afc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8709afc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8709afc

Branch: refs/heads/yang22-hbase1.x
Commit: a8709afc2b40718897c4e4f1d0a9b9f71227eeb0
Parents: ea778d2
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Mon Apr 10 18:39:00 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[4/8] kylin git commit: support azure hdinsight counters

2017-04-10 Thread billyliu

support azure hdinsight counters


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09e75450
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09e75450
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09e75450

Branch: refs/heads/yang22-hbase1.x
Commit: 09e75450879281b5921ffa24cd5fc6edc554ee73
Parents: 6617955
Author: shaofengshi 
Authored: Wed Mar 8 11:23:29 2017 +0800
Committer: shaofengshi 
Committed: Sat Apr 1 08:56:09 2017 +0800

--
 .../kylin/engine/mr/common/HadoopCmdOutput.java   | 14 --
 1 file changed, 12 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/09e75450/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 9d016cc..023717f 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -22,6 +22,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskCounter;
@@ -93,9 +94,18 @@ public class HadoopCmdOutput {
 this.output.append(counters.toString()).append("\n");
 logger.debug(counters.toString());
 
+String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN";
+String bytsReadCounterName = "HDFS_BYTES_READ";
+String fsScheme = 
FileSystem.get(job.getConfiguration()).getScheme();
+if (("wasb").equalsIgnoreCase(fsScheme)) {
+// for Azure blob store
+bytsWrittenCounterName = "WASB_BYTES_WRITTEN";
+bytsReadCounterName = "WASB_BYTES_READ";
+}
+
 mapInputRecords = 
String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
-hdfsBytesWritten = 
String.valueOf(counters.findCounter("FileSystemCounters", 
"HDFS_BYTES_WRITTEN").getValue());
-hdfsBytesRead = 
String.valueOf(counters.findCounter("FileSystemCounters", 
"HDFS_BYTES_READ").getValue());
+hdfsBytesWritten = 
String.valueOf(counters.findCounter("FileSystemCounters", 
bytsWrittenCounterName).getValue());
+hdfsBytesRead = 
String.valueOf(counters.findCounter("FileSystemCounters", 
bytsReadCounterName).getValue());
 } catch (Exception e) {
 logger.error(e.getLocalizedMessage(), e);
 output.append(e.getLocalizedMessage());

[5/8] kylin git commit: KYLIN-2537 hbase r/w bug fix

2017-04-10 Thread billyliu

KYLIN-2537 hbase r/w bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c82a0d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c82a0d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c82a0d0

Branch: refs/heads/yang22-hbase1.x
Commit: 9c82a0d0ec138efc04c5393ce475910b887ad78f
Parents: 09e7545
Author: Li Yang 
Authored: Mon Apr 10 14:12:18 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:21:20 2017 +0800

--
 .../java/org/apache/kylin/storage/hbase/HBaseConnection.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c82a0d0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 335bfe7..d7311a9 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -202,10 +202,13 @@ public class HBaseConnection {
 conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, 
nameServices.toArray(new String[0]));
 }
 
-public static String makeQualifiedPathInHBaseCluster(String path) {
+public static String makeQualifiedPathInHBaseCluster(String inPath) {
+Path path = new Path(inPath);
+path = Path.getPathWithoutSchemeAndAuthority(path);
+
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-return fs.makeQualified(new Path(path)).toString();
+return fs.makeQualified(path).toString();
 } catch (IOException e) {
 throw new IllegalArgumentException("Cannot create FileSystem from 
current hbase cluster conf", e);
 }

[1/8] kylin git commit: KYLIN-2497 fix minor bug to support long type [Forced Update!]

2017-04-10 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 80f0a0f82 -> a8709afc2 (forced update)


KYLIN-2497 fix minor bug to support long type


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50955963
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50955963
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50955963

Branch: refs/heads/yang22-hbase1.x
Commit: 50955963d117a8a0a05b756cecbb6b1c6d06647d
Parents: e477b51
Author: Hongbin Ma 
Authored: Sat Mar 11 16:58:06 2017 +0800
Committer: Hongbin Ma 
Committed: Sat Mar 11 16:58:06 2017 +0800

--
 .../org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/50955963/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index acec972..bb4152e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -50,7 +50,7 @@ public class SelfDefineSortableKey implements 
WritableComparable

[7/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-10 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a8709afc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[3/8] kylin git commit: KYLIN-2528 refine job email notification to support starttls and customized port

2017-04-10 Thread billyliu

KYLIN-2528 refine job email notification to support starttls and customized port


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6617955b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6617955b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6617955b

Branch: refs/heads/yang22-hbase1.x
Commit: 6617955b4fe065af9da46b2a9bc665a466e0
Parents: 0c6441c
Author: Hongbin Ma 
Authored: Fri Mar 31 21:15:00 2017 +0800
Committer: Hongbin Ma 
Committed: Fri Mar 31 21:15:00 2017 +0800

--
 build/conf/kylin.properties  | 13 -
 .../org/apache/kylin/common/KylinConfigBase.java |  9 +
 .../org/apache/kylin/common/util/MailService.java| 15 +--
 examples/test_case_data/localmeta/kylin.properties   | 15 +--
 examples/test_case_data/sandbox/kylin.properties |  9 -
 5 files changed, 47 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 5e7d235..7b93c94 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -109,11 +109,14 @@ kylin.job.status.with.kerberos=false
 kylin.job.step.timeout=7200
 
 # If true, will send email notification on job complete
-kylin.job.notification-enabled=false
-kylin.job.notification-mail-host=
-kylin.job.notification-mail-username=
-kylin.job.notification-mail-password=
-kylin.job.notification-mail-sender=
+#kylin.job.notification-enabled=true
+#kylin.job.notification-mail-enable-starttls=true
+#kylin.job.notification-mail-host=smtp.office365.com
+#kylin.job.notification-mail-port=587
+#kylin.job.notification-mail-username=ky...@example.com
+#kylin.job.notification-mail-password=mypassword
+#kylin.job.notification-mail-sender=ky...@example.com
+
 
 
 ### ENGINE ###

http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5b7c734..ac9ed04 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -402,6 +402,15 @@ abstract public class KylinConfigBase implements 
Serializable {
 setProperty("kylin.job.notification-enabled", "" + enable);
 }
 
+public boolean isStarttlsEnabled() {
+return 
Boolean.parseBoolean(getOptional("kylin.job.notification-mail-enable-starttls", 
"false"));
+}
+
+public String getSmtpPort() {
+return getOptional("kylin.job.notification-mail-port", "25");
+}
+
+
 public String getMailHost() {
 return getOptional("kylin.job.notification-mail-host", "");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6617955b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java 
b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
index 5793967..3bccd8f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
@@ -34,18 +34,22 @@ public class MailService {
 private static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(MailService.class);
 
 private Boolean enabled = Boolean.TRUE;
+private Boolean starttlsEnabled = Boolean.FALSE;
 private String host;
+private String port;
 private String username;
 private String password;
 private String sender;
 
 public MailService(KylinConfig config) {
-this(config.isMailEnabled(), config.getMailHost(), 
config.getMailUsername(), config.getMailPassword(), config.getMailSender());
+this(config.isMailEnabled(), config.isStarttlsEnabled(), 
config.getMailHost(), config.getSmtpPort(), config.getMailUsername(), 
config.getMailPassword(), config.getMailSender());
 }
 
-private MailService(boolean enabled, String host, String username, String 
password, String sender) {
+private MailService(boolean enabled, boolean starttlsEnabled, String host, 
String port, String username, String password, String sender) {
 this.enabled = enabled;
+this.starttlsEnabled = starttlsEnabled;
 this.host = host;
+this.port =

[1/2] kylin git commit: KYLIN-2537 hbase r/w bug fix

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang22 09e754508 -> ea778d2c9


KYLIN-2537 hbase r/w bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c82a0d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c82a0d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c82a0d0

Branch: refs/heads/yang22
Commit: 9c82a0d0ec138efc04c5393ce475910b887ad78f
Parents: 09e7545
Author: Li Yang 
Authored: Mon Apr 10 14:12:18 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:21:20 2017 +0800

--
 .../java/org/apache/kylin/storage/hbase/HBaseConnection.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9c82a0d0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 335bfe7..d7311a9 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -202,10 +202,13 @@ public class HBaseConnection {
 conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, 
nameServices.toArray(new String[0]));
 }
 
-public static String makeQualifiedPathInHBaseCluster(String path) {
+public static String makeQualifiedPathInHBaseCluster(String inPath) {
+Path path = new Path(inPath);
+path = Path.getPathWithoutSchemeAndAuthority(path);
+
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-return fs.makeQualified(new Path(path)).toString();
+return fs.makeQualified(path).toString();
 } catch (IOException e) {
 throw new IllegalArgumentException("Cannot create FileSystem from 
current hbase cluster conf", e);
 }

[2/2] kylin git commit: KYLIN-2537 fix DeployCoprocessorCLI

2017-04-10 Thread liyang

KYLIN-2537 fix DeployCoprocessorCLI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea778d2c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea778d2c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea778d2c

Branch: refs/heads/yang22
Commit: ea778d2c9748acfe9931bcdd91972cba48630f9c
Parents: 9c82a0d
Author: Li Yang 
Authored: Mon Apr 10 18:07:50 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:21:24 2017 +0800

--
 .../org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea778d2c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..8d20792 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -382,6 +382,7 @@ public class DeployCoprocessorCLI {
 
 private static Path getCoprocessorHDFSDir(FileSystem fileSystem, 
KylinConfig config) throws IOException {
 String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
+hdfsWorkingDirectory = 
HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
 Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
 fileSystem.mkdirs(coprocessorDir);
 return coprocessorDir;

kylin git commit: KYLIN-2537 fix DeployCoprocessorCLI

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master c80d43249 -> c17b3eb36


KYLIN-2537 fix DeployCoprocessorCLI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c17b3eb3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c17b3eb3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c17b3eb3

Branch: refs/heads/master
Commit: c17b3eb36bd89abe0d6b479b223047b68f933cc7
Parents: c80d432
Author: Li Yang 
Authored: Mon Apr 10 18:07:50 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:20:01 2017 +0800

--
 .../org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c17b3eb3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 29c738e..d51b71e 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -420,6 +420,7 @@ public class DeployCoprocessorCLI {
 
 private static Path getCoprocessorHDFSDir(FileSystem fileSystem, 
KylinConfig config) throws IOException {
 String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
+hdfsWorkingDirectory = 
HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
 Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
 fileSystem.mkdirs(coprocessorDir);
 return coprocessorDir;

kylin git commit: KYLIN-2537 fix DeployCoprocessorCLI

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/2.0.x a997dd66c -> 41e383c49


KYLIN-2537 fix DeployCoprocessorCLI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41e383c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41e383c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41e383c4

Branch: refs/heads/2.0.x
Commit: 41e383c49073e34952af020ee48e6281174da920
Parents: a997dd6
Author: Li Yang 
Authored: Mon Apr 10 18:07:50 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:18:57 2017 +0800

--
 .../org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/41e383c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 29c738e..d51b71e 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -420,6 +420,7 @@ public class DeployCoprocessorCLI {
 
 private static Path getCoprocessorHDFSDir(FileSystem fileSystem, 
KylinConfig config) throws IOException {
 String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
+hdfsWorkingDirectory = 
HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
 Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
 fileSystem.mkdirs(coprocessorDir);
 return coprocessorDir;

[1/2] kylin git commit: KYLIN-2537 hbase r/w bug fix [Forced Update!]

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2537 04e324f5c -> 33872bb6d (forced update)


KYLIN-2537 hbase r/w bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3da55b80
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3da55b80
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3da55b80

Branch: refs/heads/KYLIN-2537
Commit: 3da55b8012e1cefdce00a3a8f4169ab77c862c5d
Parents: 2e87fb4
Author: Li Yang 
Authored: Mon Apr 10 14:12:18 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 17:18:19 2017 +0800

--
 .../java/org/apache/kylin/storage/hbase/HBaseConnection.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3da55b80/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 73f31c5..7e2cefc 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -196,10 +196,13 @@ public class HBaseConnection {
 conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, 
nameServices.toArray(new String[0]));
 }
 
-public static String makeQualifiedPathInHBaseCluster(String path) {
+public static String makeQualifiedPathInHBaseCluster(String inPath) {
+Path path = new Path(inPath);
+path = Path.getPathWithoutSchemeAndAuthority(path);
+
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-return fs.makeQualified(new Path(path)).toString();
+return fs.makeQualified(path).toString();
 } catch (IOException e) {
 throw new IllegalArgumentException("Cannot create FileSystem from 
current hbase cluster conf", e);
 }

[2/2] kylin git commit: further fix

2017-04-10 Thread liyang

further fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/33872bb6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/33872bb6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/33872bb6

Branch: refs/heads/KYLIN-2537
Commit: 33872bb6d151d6c11dc6925332eafa12d598821d
Parents: 3da55b8
Author: Li Yang 
Authored: Mon Apr 10 18:07:50 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 18:07:50 2017 +0800

--
 .../org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/33872bb6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 29c738e..d51b71e 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -420,6 +420,7 @@ public class DeployCoprocessorCLI {
 
 private static Path getCoprocessorHDFSDir(FileSystem fileSystem, 
KylinConfig config) throws IOException {
 String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
+hdfsWorkingDirectory = 
HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
 Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
 fileSystem.mkdirs(coprocessorDir);
 return coprocessorDir;

kylin git commit: KYLIN-2537 hbase r/w bug fix

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master b641032bf -> c80d43249


KYLIN-2537 hbase r/w bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c80d4324
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c80d4324
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c80d4324

Branch: refs/heads/master
Commit: c80d4324962a5152380fbdb6b65980331ed9c1ab
Parents: b641032
Author: Li Yang 
Authored: Mon Apr 10 14:12:18 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 17:20:30 2017 +0800

--
 .../java/org/apache/kylin/storage/hbase/HBaseConnection.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c80d4324/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 73f31c5..7e2cefc 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -196,10 +196,13 @@ public class HBaseConnection {
 conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, 
nameServices.toArray(new String[0]));
 }
 
-public static String makeQualifiedPathInHBaseCluster(String path) {
+public static String makeQualifiedPathInHBaseCluster(String inPath) {
+Path path = new Path(inPath);
+path = Path.getPathWithoutSchemeAndAuthority(path);
+
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-return fs.makeQualified(new Path(path)).toString();
+return fs.makeQualified(path).toString();
 } catch (IOException e) {
 throw new IllegalArgumentException("Cannot create FileSystem from 
current hbase cluster conf", e);
 }

kylin git commit: KYLIN-2537 hbase r/w bug fix

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/2.0.x a2c4be6e8 -> a997dd66c


KYLIN-2537 hbase r/w bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a997dd66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a997dd66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a997dd66

Branch: refs/heads/2.0.x
Commit: a997dd66c52a357bcd366f0de041b616f4e1
Parents: a2c4be6
Author: Li Yang 
Authored: Mon Apr 10 14:12:18 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 17:19:44 2017 +0800

--
 .../java/org/apache/kylin/storage/hbase/HBaseConnection.java  | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a997dd66/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 73f31c5..7e2cefc 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -196,10 +196,13 @@ public class HBaseConnection {
 conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, 
nameServices.toArray(new String[0]));
 }
 
-public static String makeQualifiedPathInHBaseCluster(String path) {
+public static String makeQualifiedPathInHBaseCluster(String inPath) {
+Path path = new Path(inPath);
+path = Path.getPathWithoutSchemeAndAuthority(path);
+
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-return fs.makeQualified(new Path(path)).toString();
+return fs.makeQualified(path).toString();
 } catch (IOException e) {
 throw new IllegalArgumentException("Cannot create FileSystem from 
current hbase cluster conf", e);
 }

kylin git commit: KYLIN-2537 further bug fix

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2537 be10e237b -> 04e324f5c


KYLIN-2537 further bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/04e324f5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/04e324f5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/04e324f5

Branch: refs/heads/KYLIN-2537
Commit: 04e324f5c7d93cb8adf66bac715d658f0cc32146
Parents: be10e23
Author: Li Yang 
Authored: Mon Apr 10 15:19:40 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 15:19:40 2017 +0800

--
 .../apache/kylin/storage/hbase/HBaseConnection.java  | 15 +++
 1 file changed, 3 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/04e324f5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 9e4238d..7e2cefc 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -197,21 +197,12 @@ public class HBaseConnection {
 }
 
 public static String makeQualifiedPathInHBaseCluster(String inPath) {
-String path = inPath;
-
-// chop off schema
-int schemaCut = path.indexOf("://");
-if (schemaCut > 0) {
-path = path.substring(schemaCut + "://".length());
-}
-
-// assert absolute path
-if (path.startsWith("/") == false)
-throw new IllegalArgumentException("Expect an absolute path but 
got: " + inPath);
+Path path = new Path(inPath);
+path = Path.getPathWithoutSchemeAndAuthority(path);
 
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
-return fs.makeQualified(new Path(path)).toString();
+return fs.makeQualified(path).toString();
 } catch (IOException e) {
 throw new IllegalArgumentException("Cannot create FileSystem from 
current hbase cluster conf", e);
 }

kylin git commit: KYLIN-2537 hbase r/w bug fix

2017-04-10 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2537 [created] be10e237b


KYLIN-2537 hbase r/w bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be10e237
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be10e237
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be10e237

Branch: refs/heads/KYLIN-2537
Commit: be10e237bcd3b8b675264d212125c4a638525d4a
Parents: 2e87fb4
Author: Li Yang 
Authored: Mon Apr 10 14:12:18 2017 +0800
Committer: Li Yang 
Committed: Mon Apr 10 14:12:18 2017 +0800

--
 .../apache/kylin/storage/hbase/HBaseConnection.java   | 14 +-
 1 file changed, 13 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/be10e237/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 73f31c5..9e4238d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -196,7 +196,19 @@ public class HBaseConnection {
 conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, 
nameServices.toArray(new String[0]));
 }
 
-public static String makeQualifiedPathInHBaseCluster(String path) {
+public static String makeQualifiedPathInHBaseCluster(String inPath) {
+String path = inPath;
+
+// chop off schema
+int schemaCut = path.indexOf("://");
+if (schemaCut > 0) {
+path = path.substring(schemaCut + "://".length());
+}
+
+// assert absolute path
+if (path.startsWith("/") == false)
+throw new IllegalArgumentException("Expect an absolute path but 
got: " + inPath);
+
 try {
 FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
 return fs.makeQualified(new Path(path)).toString();

96 matches

Mail list logo