kylin git commit: KYLIN-2255 drop v1 coprocessor in DeployCoprocessorCLI

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 8b6462063 -> af8940d5b


KYLIN-2255 drop v1 coprocessor in DeployCoprocessorCLI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af8940d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af8940d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af8940d5

Branch: refs/heads/master
Commit: af8940d5b38087f71edfe4d3eb4bb32af1177bab
Parents: 8b64620
Author: Li Yang 
Authored: Thu Dec 8 15:06:04 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 15:06:04 2016 +0800

--
 .../apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af8940d5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..c8410f9 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -64,9 +64,9 @@ import com.google.common.collect.Lists;
  */
 public class DeployCoprocessorCLI {
 
-public static final String CubeObserverClass = 
"org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.AggregateRegionObserver";
 public static final String CubeEndpointClass = 
"org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService";
 public static final String CubeObserverClassOld = 
"org.apache.kylin.storage.hbase.coprocessor.observer.AggregateRegionObserver";
+public static final String CubeObserverClassOld2 = 
"org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.AggregateRegionObserver";
 public static final String IIEndpointClassOld = 
"org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint";
 public static final String IIEndpointClass = 
"org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.IIEndpoint";
 
@@ -187,7 +187,6 @@ public class DeployCoprocessorCLI {
 public static void addCoprocessorOnHTable(HTableDescriptor desc, Path 
hdfsCoprocessorJar) throws IOException {
 logger.info("Add coprocessor on " + desc.getNameAsString());
 desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
-desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
 }
 
 public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
@@ -207,8 +206,8 @@ public class DeployCoprocessorCLI {
 logger.info("Disable " + tableName);
 hbaseAdmin.disableTable(tableName);
 
-while (desc.hasCoprocessor(CubeObserverClass)) {
-desc.removeCoprocessor(CubeObserverClass);
+while (desc.hasCoprocessor(CubeObserverClassOld2)) {
+desc.removeCoprocessor(CubeObserverClassOld2);
 }
 while (desc.hasCoprocessor(CubeEndpointClass)) {
 desc.removeCoprocessor(CubeEndpointClass);

kylin git commit: minor but important, ColumnDesc.equals() check only table and name

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22 ac621fea4 -> 55f87f477


minor but important, ColumnDesc.equals() check only table and name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55f87f47
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55f87f47
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55f87f47

Branch: refs/heads/yang22
Commit: 55f87f477ee608880c7e4fcd5c6c051d27e60d51
Parents: ac621fe
Author: Li Yang 
Authored: Thu Dec 8 14:58:46 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 14:58:46 2016 +0800

--
 .../main/java/org/apache/kylin/metadata/model/ColumnDesc.java  | 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/55f87f47/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 403eaaf..2da1f5e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -197,12 +197,6 @@ public class ColumnDesc implements Serializable {
 } else if (!table.getIdentity().equals(other.table.getIdentity()))
 return false;
 
-if (datatype == null) {
-if (other.datatype != null)
-return false;
-} else if (!datatype.equals(other.datatype))
-return false;
-
 return true;
 }
 

kylin git commit: minor but important, ColumnDesc.equals() check only table and name

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 4c39fcb0e -> 8b6462063


minor but important, ColumnDesc.equals() check only table and name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8b646206
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8b646206
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8b646206

Branch: refs/heads/master
Commit: 8b64620631200a724117dec97e3c69c19d9cf22c
Parents: 4c39fcb
Author: Li Yang 
Authored: Thu Dec 8 14:48:11 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 14:48:11 2016 +0800

--
 .../main/java/org/apache/kylin/metadata/model/ColumnDesc.java  | 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8b646206/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 403eaaf..2da1f5e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -197,12 +197,6 @@ public class ColumnDesc implements Serializable {
 } else if (!table.getIdentity().equals(other.table.getIdentity()))
 return false;
 
-if (datatype == null) {
-if (other.datatype != null)
-return false;
-} else if (!datatype.equals(other.datatype))
-return false;
-
 return true;
 }
 

[13/19] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
deleted file mode 100644
index a900ea1..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.gridtable.StorageSideBehavior;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-
-/**
- * @author yangli9
- * 
- */
-public class AggregationScanner implements RegionScanner {
-
-private RegionScanner outerScanner;
-private StorageSideBehavior behavior;
-
-public AggregationScanner(CoprocessorRowType type, CoprocessorFilter 
filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner 
innerScanner, StorageSideBehavior behavior) throws IOException {
-
-AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
-
-this.behavior = behavior;
-
-ObserverAggregationCache aggCache;
-Stats stats = new Stats();
-
-aggCache = buildAggrCache(innerScanner, type, groupBy, aggrs, filter, 
stats);
-stats.countOutputRow(aggCache.getSize());
-this.outerScanner = aggCache.getScanner(innerScanner);
-
-AggregateRegionObserver.LOG.info("Kylin Coprocessor aggregation done: 
" + stats);
-}
-
-@SuppressWarnings("rawtypes")
-ObserverAggregationCache buildAggrCache(final RegionScanner innerScanner, 
CoprocessorRowType type, CoprocessorProjector projector, ObserverAggregators 
aggregators, CoprocessorFilter filter, Stats stats) throws IOException {
-
-ObserverAggregationCache aggCache = new 
ObserverAggregationCache(aggregators);
-
-ObserverTuple tuple = new ObserverTuple(type);
-boolean hasMore = true;
-List results = new ArrayList();
-byte meaninglessByte = 0;
-
-while (hasMore) {
-results.clear();
-hasMore = innerScanner.nextRaw(results);
-if (results.isEmpty())
-continue;
-
-if (stats != null)
-stats.countInputRow(results);
-
-Cell cell = results.get(0);
-tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), 
cell.getRowLength());
-
-if (behavior == StorageSideBehavior.SCAN) {
-//touch every byte of the cell so that the cost of scanning 
will be trully reflected
-int endIndex = cell.getRowOffset() + cell.getRowLength();
-for (int i = cell.getRowOffset(); i < endIndex; ++i) {
-meaninglessByte += cell.getRowArray()[i];
-}
-} else {
-if (behavior.filterToggledOn()) {
-if (filter != null && filter.evaluate(tuple) == false)
-continue;
-
-if (behavior.aggrToggledOn()) {
-AggrKey aggKey = projector.getAggrKey(results);
-MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
-aggregators.aggregate(bufs, results);
-
-if (behavior.ordinal() >= 

[15/19] kylin git commit: KYLIN-2217 fix null partition col bug

[liyang]

KYLIN-2217 fix null partition col bug

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c56f4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c56f4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c56f4b

Branch: refs/heads/master-cdh5.7
Commit: c0c56f4b6b05bb108a4952a56af992b237c1bbe6
Parents: 545201f
Author: xiefan46 <958034...@qq.com>
Authored: Thu Dec 8 10:33:48 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:09:31 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java| 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c56f4b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8933ee2..a3e61a1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -116,13 +116,17 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

[09/19] kylin git commit: minor, fix date format in sample data

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/21bcd2f6/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index e13b7ad..215 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,14/12/2012,Others,88750,0,11,36.2828,4,1349,10002313,ANALYST,Beijing
-1,28/08/2012,Others,175750,0,13,23.8563,20,1927,10004376,ANALYST,Beijing
-2,16/02/2012,ABIN,148324,15,13,88.3418,18,1005,10006710,ADMIN,Shanghai
-3,19/10/2013,FP-non GTC,37831,0,13,47.3015,3,1209,10003717,ANALYST,Beijing
-4,22/10/2012,Others,140746,100,11,83.454,16,1154,10006076,ADMIN,Shanghai
-5,25/01/2013,FP-GTC,16509,0,-99,22.9896,11,1372,10007436,ADMIN,Shanghai
-6,04/04/2013,Others,963,0,13,88.5907,11,1648,10009869,MODELER,Hongkong
-7,11/04/2012,Others,15687,0,15,88.194,19,1866,1400,ADMIN,Shanghai
-8,19/01/2013,ABIN,60606,3,13,77.9727,13,1936,1687,MODELER,Hongkong
-9,30/04/2012,FP-non 
GTC,106246,0,14,52.295,17,1047,10009223,MODELER,Hongkong
-10,03/02/2013,Auction,45333,0,16,56.3584,10,1470,10006759,MODELER,Hongkong
-11,02/09/2012,FP-GTC,25147,0,-99,15.3553,8,1203,1062,ANALYST,Beijing
-12,30/07/2013,Others,24760,0,16,25.077,5,1382,10003539,ADMIN,Shanghai
-13,17/04/2012,Auction,31519,0,16,72.7384,11,1069,10001245,ADMIN,Shanghai
-14,30/04/2013,FP-GTC,51582,0,15,75.82,6,1085,10008148,ANALYST,Beijing
-15,23/03/2013,FP-GTC,16509,0,15,8.6653,15,1559,10008079,MODELER,Hongkong
-16,01/04/2013,ABIN,175750,3,16,64.2802,16,1507,10007797,MODELER,Hongkong
-17,12/07/2013,ABIN,50677,0,-99,24.5987,17,1938,10006224,ADMIN,Shanghai
-18,23/04/2012,FP-GTC,45333,0,5,72.6553,15,1863,10003824,MODELER,Hongkong
-19,19/12/2012,ABIN,3838,0,12,87.4142,4,1612,10007157,ADMIN,Shanghai
-20,15/08/2012,FP-GTC,62179,0,16,67.4238,15,1839,10003452,MODELER,Hongkong
-21,22/08/2013,Auction,1357,0,5,18.36,9,1303,1114,ANALYST,Beijing
-22,23/08/2013,ABIN,31519,0,5,67.6344,2,1877,10007417,ANALYST,Beijing
-23,12/09/2013,FP-GTC,223,0,14,71.7898,15,1848,1627,MODELER,Hongkong
-24,14/03/2012,FP-GTC,132939,0,15,26.1496,20,1815,10005292,MODELER,Hongkong
-25,13/07/2013,Others,67703,3,16,83.9661,2,1858,10008123,MODELER,Hongkong
-26,15/12/2013,FP-non GTC,100847,0,5,34.3451,16,1387,10007806,ADMIN,Shanghai
-27,30/08/2012,Auction,62179,0,11,21.0496,9,1393,10007360,ANALYST,Beijing
-28,10/01/2013,FP-GTC,48027,0,13,23.4142,20,1757,10002135,MODELER,Hongkong
-29,25/01/2013,ABIN,145970,0,5,26.7842,6,1402,10009682,ANALYST,Beijing
-30,05/07/2012,Auction,48027,0,-99,46.732,5,1972,10009522,MODELER,Hongkong
-31,07/11/2012,FP-GTC,20213,0,-99,19.727,8,1873,10009944,ANALYST,Beijing
-32,20/06/2013,ABIN,100847,0,5,76.2783,1,1409,10005395,ADMIN,Shanghai
-33,08/10/2012,ABIN,26262,0,5,48.7447,4,1029,10001836,ADMIN,Shanghai
-34,30/04/2013,ABIN,87118,0,15,63.2187,11,1529,1928,ANALYST,Beijing
-35,24/04/2013,FP-non 
GTC,20213,0,16,39.5057,19,1329,1768,ANALYST,Beijing
-36,13/05/2012,Others,164261,0,13,84.1246,15,1688,1727,ADMIN,Shanghai
-37,02/08/2013,ABIN,31519,0,5,37.1504,14,1905,1719,MODELER,Hongkong
-38,03/04/2012,Auction,164261,0,11,92.0974,13,1341,10001337,ADMIN,Shanghai
-39,02/09/2012,FP-GTC,31519,0,14,87.7829,20,1460,10008484,ANALYST,Beijing
-40,05/10/2013,ABIN,13987,0,16,74.0719,4,1381,10002975,MODELER,Hongkong
-41,13/01/2012,FP-GTC,48027,0,15,71.2637,14,1915,10005778,ADMIN,Shanghai
-42,01/03/2013,Others,13836,0,14,16.7288,14,1953,10009385,ADMIN,Shanghai
-43,12/09/2012,FP-non 
GTC,16509,0,5,12.2933,20,1277,10002286,MODELER,Hongkong
-44,07/02/2012,Auction,45333,0,5,64.977,16,1040,10005583,MODELER,Hongkong
-45,14/12/2013,FP-non 
GTC,158798,0,16,72.4413,18,1500,10007635,MODELER,Hongkong
-46,13/10/2013,Auction,31519,0,5,79.3053,17,1816,10006423,MODELER,Hongkong
-47,11/06/2012,ABIN,4943,0,5,11.6942,15,1696,10001237,ANALYST,Beijing
-48,18/10/2012,ABIN,80053,0,-99,54.0933,4,1604,10006851,MODELER,Hongkong
-49,03/11/2012,Auction,20886,0,5,9.8258,8,1764,10001220,MODELER,Hongkong
-50,15/01/2012,Auction,44079,0,14,13.0371,19,1343,10001259,MODELER,Hongkong
-51,03/07/2012,FP-GTC,159184,0,-99,92.5314,4,1724,10009008,ADMIN,Shanghai
-52,16/02/2012,FP-GTC,314,0,13,49.1825,20,1334,10004005,ADMIN,Shanghai
-53,17/06/2012,FP-non GTC,31387,3,12,0.6677,5,1603,10002595,ANALYST,Beijing
-54,15/07/2012,FP-non 
GTC,32876,0,-99,50.9634,11,1248,10006890,ANALYST,Beijing
-55,24/07/2012,FP-GTC,150265,15,16,57.5645,6,1653,10007269,ADMIN,Shanghai
-56,06/10/2012,Auction,145970,0,5,44.3091,16,1082,10001640,ADMIN,Shanghai
-57,14/06/2013,ABIN,80053,0,5,20.2388,19,1632,10002068,MODELER,Hongkong
-58,10/10/2013,FP-non 

[18/19] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eaf0cc58
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eaf0cc58
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eaf0cc58

Branch: refs/heads/master-cdh5.7
Commit: eaf0cc5866367eadacfb7a43ca793c2f585dfdc6
Parents: 4c39fcb
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 13:42:27 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 40 files changed, 365 insertions(+), 432 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 67b62d5..8bd1d67 

[03/19] kylin git commit: KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc

[liyang]

KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64d9b8b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64d9b8b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64d9b8b8

Branch: refs/heads/master-cdh5.7
Commit: 64d9b8b89f84cc31f9931e5a6cdf89bdfa13925a
Parents: 227bbf4
Author: Li Yang 
Authored: Thu Dec 1 14:10:44 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:37 2016 +0800

--
 .../apache/kylin/cube/RawQueryLastHacker.java   |  8 +---
 .../metadata/measure/MeasureCodecTest.java  |  3 +-
 .../kylin/measure/raw/RawMeasureType.java   | 12 ++---
 .../kylin/metadata/model/FunctionDesc.java  | 26 ---
 .../kylin/metadata/model/ParameterDesc.java | 47 +++-
 .../apache/kylin/storage/StorageMockUtils.java  | 32 +++--
 .../kylin/query/relnode/OLAPAggregateRel.java   | 12 ++---
 7 files changed, 59 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index b0a4823..682e48c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -57,12 +57,8 @@ public class RawQueryLastHacker {
 sqlDigest.groupbyColumns.add(col);
 } else {
 // For measure columns, take them as metric columns with 
aggregation function SUM().
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc sumFunc = new FunctionDesc();
-sumFunc.setExpression("SUM");
-sumFunc.setParameter(colParameter);
+ParameterDesc parameter = ParameterDesc.newInstance(col);
+FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", 
parameter, null);
 
 boolean measureHasSum = false;
 for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 6bb71e9..18680ec 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -83,8 +83,7 @@ public class MeasureCodecTest extends 
LocalFileMetadataTestCase {
 
 private MeasureDesc measure(String returnType) {
 MeasureDesc desc = new MeasureDesc();
-FunctionDesc func = new FunctionDesc();
-func.setReturnType(returnType);
+FunctionDesc func = FunctionDesc.newInstance(null, null, returnType);
 desc.setFunction(func);
 return desc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 3a49d31..a5bb06b 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -197,18 +197,12 @@ public class RawMeasureType extends 
MeasureType {
 if (sqlDigest.isRawQuery) {
 for (MeasureDesc measureDesc : measureDescs) {
 TblColRef col = this.getRawColumn(measureDesc.getFunction());
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc rawFunc = new FunctionDesc();
-rawFunc.setExpression("RAW");
-rawFunc.setParameter(colParameter);
+ParameterDesc colParameter = ParameterDesc.newInstance(col);
+FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", 
colParameter, null);
 
 if 

[01/19] kylin git commit: KYLIN-1875 Normalize table and column names in JSON [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master-cdh5.7 db1b826ac -> bfc89a0f7 (forced update)


KYLIN-1875 Normalize table and column names in JSON


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/967ef180
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/967ef180
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/967ef180

Branch: refs/heads/master-cdh5.7
Commit: 967ef18062048e199cd0dd351ba82618a10b08e5
Parents: 081ed0d
Author: Li Yang 
Authored: Tue Nov 29 20:32:04 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:35 2016 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   | 22 +---
 .../org/apache/kylin/cube/model/CubeDesc.java   |  1 +
 .../apache/kylin/cube/model/DimensionDesc.java  | 19 +-
 .../model/validation/rule/FunctionRule.java | 37 
 .../kylin/metadata/model/DataModelDesc.java | 13 ++-
 .../kylin/metadata/model/FunctionDesc.java  |  5 +--
 .../metadata/model/ModelDimensionDesc.java  | 19 +++---
 .../kylin/metadata/model/PartitionDesc.java |  4 +--
 8 files changed, 64 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 677b713..011035b 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -81,8 +81,7 @@ public class FactTableGenerator {
 // table(appear as fk in fact table)
 TreeMap lookupTableKeys = new 
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
-// possible values of lookupTableKeys, extracted from existing lookup
-// tables.
+// possible values of lookupTableKeys, extracted from existing lookup 
tables.
 // The key is in the format of tablename/columnname
 TreeMap feasibleValues = new 
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
@@ -244,7 +243,7 @@ public class FactTableGenerator {
 JoinDesc join = dim.getJoin();
 if (join != null) {
 String lookupTable = dim.getTableRef().getTableIdentity();
-for (String column : join.getPrimaryKey()) {
+for (String column : dropAlias(join.getPrimaryKey())) {
 if (!lookupTableKeys.containsKey(lookupTable)) {
 lookupTableKeys.put(lookupTable, new 
LinkedList());
 }
@@ -297,8 +296,8 @@ public class FactTableGenerator {
 for (DimensionDesc dim : dimensions) {
 JoinDesc jDesc = dim.getJoin();
 if (jDesc != null) {
-String[] fks = jDesc.getForeignKey();
-String[] pks = jDesc.getPrimaryKey();
+String[] fks = dropAlias(jDesc.getForeignKey());
+String[] pks = dropAlias(jDesc.getPrimaryKey());
 int num = fks.length;
 for (int i = 0; i < num; ++i) {
 String value = dim.getTableRef().getTableIdentity() + "/" 
+ pks[i];
@@ -343,6 +342,19 @@ public class FactTableGenerator {
 return createTable(this.rowCount, factTableCol2LookupCol, 
lookupCol2factTableCol, usedCols);
 }
 
+private String[] dropAlias(String[] aliasDotCol) {
+String[] result = new String[aliasDotCol.length];
+for (int i = 0; i < aliasDotCol.length; i++) {
+String str = aliasDotCol[i];
+int cut = str.lastIndexOf('.');
+if (cut >= 0) {
+str = str.substring(cut + 1);
+}
+result[i] = str;
+}
+return result;
+}
+
 private String normToTwoDigits(int v) {
 if (v < 10)
 return "0" + v;

http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 853571c..f95cceb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -543,6 +543,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 initMeasureColumns();
 
 rowkey.init(this);
+
 validateAggregationGroups(); // 

[17/19] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..8f7430e 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -190,7 +192,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -205,7 +207,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClass)) {
 desc.removeCoprocessor(CubeObserverClass);
@@ -231,16 +233,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -261,12 +262,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch 

[06/19] kylin git commit: KYLIN-1875 minor, test fixes

[liyang]

KYLIN-1875 minor, test fixes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/365caaeb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/365caaeb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/365caaeb

Branch: refs/heads/master-cdh5.7
Commit: 365caaebd841917f161dab37b0af7c055626194c
Parents: 92e4d46
Author: Yang Li 
Authored: Fri Dec 2 06:32:25 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:46 2016 +0800

--
 .../metadata/filter/ColumnTupleFilter.java  |  1 +
 .../kylin/metadata/model/DataModelDesc.java |  2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |  2 +-
 .../template/model_desc/kylin_sales_model.json  |  4 +++
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../resources/query/sql_subquery/query02.sql| 31 
 .../query/sql_subquery/query02.sql.disable  | 25 
 7 files changed, 39 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index c8a8f07..ecb8e61 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -121,6 +121,7 @@ public class ColumnTupleFilter extends TupleFilter {
 }
 }
 
+@SuppressWarnings("deprecation")
 @Override
 public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 898ff74..3f868a2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -112,7 +112,7 @@ public class DataModelDesc extends RootPersistentEntity {
 return name;
 }
 
-// for test only
+// for test mainly
 @Deprecated
 public void setName(String name) {
 this.name = name;

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index bf8d36b..5d72c3f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -84,7 +84,7 @@ public class TblColRef implements Serializable {
 col.table = tableRef;
 }
 
-// for test only
+// for test mainly
 public static TblColRef mockup(TableDesc table, int oneBasedColumnIndex, 
String name, String datatype) {
 ColumnDesc desc = new ColumnDesc();
 String id = "" + oneBasedColumnIndex;

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 2d3cfb6..cce360f 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -18,6 +18,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_ACCOUNT",
 "alias" : "BUYER_ACCOUNT",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "ACCOUNT_ID" ],
@@ -26,6 +27,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_ACCOUNT",
 "alias" : "SELLER_ACCOUNT",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "ACCOUNT_ID" ],
@@ -34,6 +36,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_COUNTRY",
 "alias" : "BUYER_COUNTRY",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "COUNTRY" ],
@@ -42,6 +45,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_COUNTRY",
 "alias" : "SELLER_COUNTRY",
+"kind" : "LOOKUP",

[05/19] kylin git commit: KYLIN-1875 fix PK-FK derive from each other

[liyang]

KYLIN-1875 fix PK-FK derive from each other


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92e4d464
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92e4d464
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92e4d464

Branch: refs/heads/master-cdh5.7
Commit: 92e4d464069745f0c58cc0559e054efa470960c1
Parents: 79e4801
Author: Yang Li 
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:39 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  8 +--
 .../org/apache/kylin/cube/model/CubeDesc.java   | 52 +++-
 .../storage/gtrecord/CubeTupleConverter.java|  2 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  2 +-
 .../translate/DerivedFilterTranslator.java  |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../query/enumerator/LookupTableEnumerator.java |  2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  2 +-
 .../hbase/cube/v1/CubeTupleConverter.java   |  2 +-
 9 files changed, 41 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index bda1423..fe2030a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -44,7 +44,6 @@ import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -54,6 +53,7 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -629,10 +629,10 @@ public class CubeManager implements IRealizationProvider {
 cubeMap.removeLocal(cubeName);
 }
 
-public LookupStringTable getLookupTable(CubeSegment cubeSegment, 
DimensionDesc dim) {
+public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc 
join) {
 
-String tableName = dim.getTableRef().getTableIdentity();
-String[] pkCols = dim.getJoin().getPrimaryKey();
+String tableName = join.getPKSide().getTableIdentity();
+String[] pkCols = join.getPrimaryKey();
 String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
 if (snapshotResPath == null)
 throw new IllegalStateException("No snaphot for table '" + 
tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() 
+ "/" + cubeSegment);

http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f95cceb..f8c316c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -101,20 +102,20 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public static class DeriveInfo {
 public DeriveType type;
-public DimensionDesc dimension;
+public JoinDesc join;
 public TblColRef[] columns;
 public boolean isOneToOne; // only used when ref from derived to host
 
-DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] 
columns, boolean isOneToOne) {
+DeriveInfo(DeriveType type, JoinDesc 

[07/19] kylin git commit: minor, add API to get AggregationGroup combination cuboid num

[liyang]

minor, add API to get AggregationGroup combination cuboid num


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1fd4679
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1fd4679
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1fd4679

Branch: refs/heads/master-cdh5.7
Commit: b1fd4679fa80e8e971d553a42baf20198afb9eac
Parents: 365caae
Author: Billy Liu 
Authored: Wed Dec 7 18:29:07 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 18:29:07 2016 +0800

--
 .../kylin/rest/controller/CubeController.java   | 33 
 1 file changed, 33 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b1fd4679/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f537231..ab32551 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -655,6 +656,22 @@ public class CubeController extends BasicController {
 return response;
 }
 
+/**
+ * Calculate Cuboid Combination based on the AggreationGroup definition.
+ *
+ * @param aggregationGroupStr
+ * @return number of cuboid, -1 if failed
+ */
+@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.POST)
+@ResponseBody
+public long calculateCuboidCombination(@RequestBody String 
aggregationGroupStr) {
+AggregationGroup aggregationGroup = 
deserializeAggregationGroup(aggregationGroupStr);
+if (aggregationGroup != null) {
+return aggregationGroup.calculateCuboidCombination();
+} else
+return -1;
+}
+
 private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
 CubeDesc desc = null;
 try {
@@ -673,6 +690,22 @@ public class CubeController extends BasicController {
 return desc;
 }
 
+private AggregationGroup deserializeAggregationGroup(String 
aggregationGroupStr) {
+AggregationGroup aggreationGroup = null;
+try {
+logger.debug("Parsing AggregationGroup " + aggregationGroupStr);
+aggreationGroup = JsonUtil.readValue(aggregationGroupStr, 
AggregationGroup.class);
+} catch (JsonParseException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (JsonMappingException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (IOException e) {
+logger.error("Failed to deal with the request.", e);
+throw new InternalErrorException("Failed to deal with the 
request:" + e.getMessage(), e);
+}
+return aggreationGroup;
+}
+
 private void updateRequest(CubeRequest request, boolean success, String 
message) {
 request.setCubeDescData("");
 request.setSuccessful(success);

[12/19] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
deleted file mode 100644
index 1bd92e6..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+++ /dev/null
@@ -1,1298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: 
storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
-
-package org.apache.kylin.storage.hbase.cube.v1.filter.generated;
-
-public final class FilterProtosExt {
-private FilterProtosExt() {
-}
-
-public static void 
registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
-}
-
-public interface BytesBytesPairOrBuilder extends 
com.google.protobuf.MessageOrBuilder {
-
-// required bytes first = 1;
-/**
- * required bytes first = 1;
- */
-boolean hasFirst();
-
-/**
- * required bytes first = 1;
- */
-com.google.protobuf.ByteString getFirst();
-
-// required bytes second = 2;
-/**
- * required bytes second = 2;
- */
-boolean hasSecond();
-
-/**
- * required bytes second = 2;
- */
-com.google.protobuf.ByteString getSecond();
-}
-
-/**
- * Protobuf type {@code BytesBytesPair}
- */
-public static final class BytesBytesPair extends 
com.google.protobuf.GeneratedMessage implements BytesBytesPairOrBuilder {
-// Use BytesBytesPair.newBuilder() to construct.
-private BytesBytesPair(com.google.protobuf.GeneratedMessage.Builder 
builder) {
-super(builder);
-this.unknownFields = builder.getUnknownFields();
-}
-
-private BytesBytesPair(boolean noInit) {
-this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance();
-}
-
-private static final BytesBytesPair defaultInstance;
-
-public static BytesBytesPair getDefaultInstance() {
-return defaultInstance;
-}
-
-public BytesBytesPair getDefaultInstanceForType() {
-return defaultInstance;
-}
-
-private final com.google.protobuf.UnknownFieldSet unknownFields;
-
-@java.lang.Override
-public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
-return this.unknownFields;
-}
-
-private BytesBytesPair(com.google.protobuf.CodedInputStream input, 
com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws 
com.google.protobuf.InvalidProtocolBufferException {
-initFields();
-int mutable_bitField0_ = 0;
-com.google.protobuf.UnknownFieldSet.Builder unknownFields = 
com.google.protobuf.UnknownFieldSet.newBuilder();
-try {
-boolean done = false;
-while (!done) {
-int tag = input.readTag();
-switch (tag) {
-case 0:
-done = true;
-break;
-default: {
-if (!parseUnknownField(input, unknownFields, 
extensionRegistry, tag)) {
-done = true;
-}
-break;
-}
-case 10: {
-bitField0_ |= 0x0001;
-first_ = input.readBytes();
-break;
-}
-case 18: {
-bitField0_ |= 0x0002;
-second_ = input.readBytes();
-break;
-}
-}
-}
-

[14/19] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

KYLIN-2255 drop v1 coprocessor impl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/545201f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/545201f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/545201f6

Branch: refs/heads/master-cdh5.7
Commit: 545201f6c0c2e4da9b3dcabac0f606e2ef880d19
Parents: 21bcd2f
Author: Li Yang 
Authored: Thu Dec 8 10:41:18 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 10:54:58 2016 +0800

--
 .../apache/kylin/query/ITCombinationTest.java   |   15 -
 .../org/apache/kylin/query/KylinTestBase.java   |2 -
 .../kylin/rest/controller/CubeController.java   |   14 -
 .../hbase/cube/v1/CubeSegmentTupleIterator.java |  344 -
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  796 ---
 .../hbase/cube/v1/CubeTupleConverter.java   |  271 
 .../hbase/cube/v1/RegionScannerAdapter.java |   97 --
 .../hbase/cube/v1/ResultScannerAdapter.java |  100 --
 .../cube/v1/SerializedHBaseTupleIterator.java   |  156 ---
 .../observer/AggregateRegionObserver.java   |  112 --
 .../observer/AggregationScanner.java|  188 ---
 .../observer/ObserverAggregationCache.java  |  166 ---
 .../observer/ObserverAggregators.java   |  265 
 .../coprocessor/observer/ObserverEnabler.java   |  191 ---
 .../v1/coprocessor/observer/ObserverTuple.java  |   71 -
 .../hbase/cube/v1/filter/FuzzyRowFilterV2.java  |  574 
 .../hbase/cube/v1/filter/UnsafeAccess.java  |  433 --
 .../v1/filter/generated/FilterProtosExt.java| 1298 --
 .../cube/v1/filter/protobuf/FilterExt.proto |   39 -
 .../cube/MeasureTypeOnlyAggrInBaseTest.java |   21 -
 .../observer/AggregateRegionObserverTest.java   |  339 -
 .../observer/RowAggregatorsTest.java|   62 -
 .../cube/v1/filter/TestFuzzyRowFilterV2.java|  249 
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  346 -
 24 files changed, 6149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 496ac4e..84573b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.query.routing.Candidate;
 import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.storage.hbase.HBaseStorage;
-import 
org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.runner.RunWith;
@@ -67,7 +66,6 @@ public class ITCombinationTest extends ITKylinQueryTest {
 public static Collection configs() {
 return Arrays.asList(new Object[][] { //
 { "inner", "on", "v2", true }, //
-{ "left", "on", "v1", true }, //
 { "left", "on", "v2", true }, //
 //{ "inner", "on", "v2", false }, // exclude view to simply 
model/cube selection
 //{ "left", "on", "v1", false }, // exclude view to simply 
model/cube selection
@@ -84,23 +82,10 @@ public class ITCombinationTest extends ITKylinQueryTest {
 ITKylinQueryTest.joinType = joinType;
 ITKylinQueryTest.setupAll();
 
-if (coprocessorToggle.equals("on")) {
-ObserverEnabler.forceCoprocessorOn();
-} else if (coprocessorToggle.equals("off")) {
-ObserverEnabler.forceCoprocessorOff();
-} else if (coprocessorToggle.equals("unset")) {
-// unset
-}
-
 RemoveBlackoutRealizationsRule.blackList.clear();
 if (excludeViewCubes) {
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
 }
-
-if ("v1".equalsIgnoreCase(queryEngine))
-HBaseStorage.overwriteStorageQuery = 
HBaseStorage.v1CubeStorageQuery;
-else
-HBaseStorage.overwriteStorageQuery = null;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java 

[08/19] kylin git commit: KYLIN-2254 allow 2 columns in a CompareTupleFilter

[liyang]

KYLIN-2254 allow 2 columns in a CompareTupleFilter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/350fad8f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/350fad8f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/350fad8f

Branch: refs/heads/master-cdh5.7
Commit: 350fad8f248988596c05b080b3c5a68b77b790a1
Parents: b1fd467
Author: Li Yang 
Authored: Wed Dec 7 17:51:26 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 09:47:27 2016 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 23 ++--
 .../kylin/metadata/model/FunctionDesc.java  |  2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |  2 +-
 .../resources/query/sql_subquery/query12.sql| 37 
 .../kylin/query/relnode/OLAPAggregateRel.java   | 13 ---
 .../kylin/query/relnode/OLAPFilterRel.java  | 30 +---
 6 files changed, 66 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..d567c89 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -37,6 +37,7 @@ public class CompareTupleFilter extends TupleFilter {
 // operand 1 is either a column or a function
 private TblColRef column;
 private FunctionTupleFilter function;
+private TblColRef secondColumn;
 
 // operand 2 is constants
 private Set conditionValues;
@@ -73,15 +74,16 @@ public class CompareTupleFilter extends TupleFilter {
 if (child instanceof ColumnTupleFilter) {
 ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
 if (this.column != null) {
-throw new IllegalStateException("Duplicate columns! old is " + 
column.getName() + " and new is " + columnFilter.getColumn().getName());
-}
-this.column = columnFilter.getColumn();
-// if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
-// children.size() > 1 means already added one conditionValue or 
dynamicVariable
-if (this.children.size() > 1 && needSwapOperator()) {
-this.operator = SWAP_OP_MAP.get(this.operator);
-TupleFilter last = this.children.remove(this.children.size() - 
1);
-this.children.add(0, last);
+this.secondColumn = columnFilter.getColumn();
+} else {
+this.column = columnFilter.getColumn();
+// if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
+// children.size() > 1 means already added one conditionValue 
or dynamicVariable
+if (this.children.size() > 1 && needSwapOperator()) {
+this.operator = SWAP_OP_MAP.get(this.operator);
+TupleFilter last = 
this.children.remove(this.children.size() - 1);
+this.children.add(0, last);
+}
 }
 } else if (child instanceof ConstantTupleFilter) {
 this.conditionValues.addAll(child.getValues());
@@ -212,7 +214,8 @@ public class CompareTupleFilter extends TupleFilter {
 
 @Override
 public boolean isEvaluable() {
-return ((function != null && function.isEvaluable()) || column != 
null) && !conditionValues.isEmpty();
+return (column != null || (function != null && 
function.isEvaluable())) //
+&& !conditionValues.isEmpty() && secondColumn == null;
 }
 
 @SuppressWarnings({ "unchecked", "rawtypes" })

http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 832cb4a..9252c42 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -140,7 +140,7 @@ public class FunctionDesc {
 } else if (isCount()) {
 return "COUNT__"; // ignores parameter, count(*), count(1), 
count(col) are all the same

[16/19] kylin git commit: KYLIN-2217 fix NPE in FactDistinctColumnsReducer

[liyang]

KYLIN-2217 fix NPE in FactDistinctColumnsReducer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c39fcb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c39fcb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c39fcb0

Branch: refs/heads/master-cdh5.7
Commit: 4c39fcb0eaab053d43554ac6add377343d585065
Parents: c0c56f4
Author: Li Yang 
Authored: Thu Dec 8 11:14:31 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:14:31 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c39fcb0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a3e61a1..776d750 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -117,7 +117,6 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

[04/19] kylin git commit: KYLIN-1875 Support measure on non-root-fact-table

[liyang]

KYLIN-1875 Support measure on non-root-fact-table


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79e48011
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79e48011
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79e48011

Branch: refs/heads/master-cdh5.7
Commit: 79e48011bd3f4696a916f0bfee244ec28355f3b3
Parents: 64d9b8b
Author: Li Yang 
Authored: Thu Dec 1 18:59:08 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:38 2016 +0800

--
 .../kylin/metadata/project/ProjectL2Cache.java  | 26 +---
 .../template/cube_desc/kylin_sales_cube.json| 18 +++---
 .../kylin/query/relnode/OLAPTableScan.java  |  4 +++
 3 files changed, 30 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 82c0de3..14d7843 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
@@ -126,17 +127,17 @@ class ProjectL2Cache {
 return Collections.unmodifiableSet(tableCache.realizations);
 }
 
-public List listEffectiveRewriteMeasures(String project, 
String factTable, boolean onlyRewriteMeasure) {
-Set realizations = getRealizationsByTable(project, 
factTable);
+public List listEffectiveRewriteMeasures(String project, 
String table, boolean onlyRewriteMeasure) {
+Set realizations = getRealizationsByTable(project, 
table);
 List result = Lists.newArrayList();
 for (IRealization r : realizations) {
-if (r.getModel().isFactTable(factTable) && r.isReady()) {
-for (MeasureDesc m : r.getMeasures()) {
-FunctionDesc func = m.getFunction();
-if (onlyRewriteMeasure) {
-if (func.needRewrite())
-result.add(m);
-} else {
+if (!r.isReady())
+continue;
+
+for (MeasureDesc m : r.getMeasures()) {
+FunctionDesc func = m.getFunction();
+if (belongToTable(func, table, r.getModel())) {
+if (!onlyRewriteMeasure || func.needRewrite()) {
 result.add(m);
 }
 }
@@ -145,6 +146,13 @@ class ProjectL2Cache {
 return result;
 }
 
+private boolean belongToTable(FunctionDesc func, String table, 
DataModelDesc model) {
+// measure belong to the first column parameter's table
+List cols = func.getParameter().getColRefs();
+String belongTo = cols.isEmpty() ? 
model.getRootFactTable().getTableIdentity() : cols.get(0).getTable();
+return belongTo.equals(table);
+}
+
 // 

 // build the cache
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 48eef46..2a27305 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -103,26 +103,26 @@
   "returntype" : "decimal(19,4)"
 }
   }, {
-"name" : "GMV_MIN",
+"name" : "BUYER_LEVEL_SUM",
 "function" : {
-  "expression" : "MIN",
+  "expression" : "SUM",
   "parameter" : {
 "type" : "column",
-"value" : "PRICE",
+"value" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
 "next_parameter" : null
   },
-  "returntype" : "decimal(19,4)"
+  "returntype" : "bigint"
 }
   }, {
-"name" : "GMV_MAX",
+"name" : "SELLER_LEVEL_SUM",
 "function" : {
-  "expression" : "MAX",
+  

[10/19] kylin git commit: minor, fix date format in sample data

[liyang]

minor, fix date format in sample data


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21bcd2f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21bcd2f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21bcd2f6

Branch: refs/heads/master-cdh5.7
Commit: 21bcd2f6b7d56e639aba18f3f376875922c9cd68
Parents: 350fad8
Author: lidongsjtu 
Authored: Thu Dec 8 10:40:45 2016 +0800
Committer: lidongsjtu 
Committed: Thu Dec 8 10:40:55 2016 +0800

--
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 1 file changed, 1 insertions(+), 1 deletions(-)
--

[11/19] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
deleted file mode 100644
index 74b2112..000
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1.filter;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestFuzzyRowFilterV2 {
-@Test
-public void testSatisfiesNoUnsafeForward() {
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, (byte) -128, 1, 0, 1 
}, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, (byte) -128, 2, 0, 1 
}, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 3, 3 }, 0, 5, 
new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
-FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 
1, 3, 0 }, // row to check
-0, 5, new byte[] { 1, 2, 0, 3 }, // fuzzy row
-new byte[] { 0, 0, 1, 0 })); // mask
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, 0, 5, 
new byte[] { 1, (byte) 245, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 0, 1 }, 0, 5, 
new byte[] { 0, 1, 2 }, new byte[] { 1, 0, 0 }));
-}
-
-@Test
-public void testSatisfiesForward() {
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, (byte) -128, 1, 0, 1 }, new 
byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, (byte) -128, 2, 0, 1 }, new 
byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 3, 3 }, new byte[] { 1, 
2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
-FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 
}, // row to check
-new byte[] { 1, 2, 0, 3 }, // fuzzy row
-new byte[] { -1, -1, 0, -1 })); // mask
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, new byte[] { 1, 
(byte) 245, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 0, 1 }, new byte[] { 0, 
1, 2 }, new byte[] { 0, -1, -1 }));
-}
-
-@Test
-public void testSatisfiesReverse() {
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) -128, 1, 0, 1 }, new 
byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) -128, 2, 0, 1 }, new 
byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
-

[02/19] kylin git commit: KYLIN-1875 fix ParameterDesc equals() & hashcode()

[liyang]

KYLIN-1875 fix ParameterDesc equals() & hashcode()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/227bbf41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/227bbf41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/227bbf41

Branch: refs/heads/master-cdh5.7
Commit: 227bbf4152074c0621bf2b1047102d416d4dd3eb
Parents: 967ef18
Author: Yang Li 
Authored: Thu Dec 1 06:58:08 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:36 2016 +0800

--
 .../kylin/metadata/model/FunctionDesc.java  |  1 +
 .../kylin/metadata/model/ParameterDesc.java | 36 +++-
 2 files changed, 28 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/227bbf41/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index dfa6f3b..b9e5543 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -248,6 +248,7 @@ public class FunctionDesc {
 final int prime = 31;
 int result = 1;
 result = prime * result + ((expression == null) ? 0 : 
expression.hashCode());
+result = prime * result + ((returnType == null) ? 0 : 
returnType.hashCode());
 result = prime * result + ((isCount() || parameter == null) ? 0 : 
parameter.hashCode());
 return result;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/227bbf41/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 329799f..4a95fea 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -22,8 +22,10 @@ import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableList;
 
 /**
  */
@@ -36,9 +38,10 @@ public class ParameterDesc {
 private String value;
 
 @JsonProperty("next_parameter")
+@JsonInclude(JsonInclude.Include.NON_NULL)
 private ParameterDesc nextParameter;
 
-private List colRefs;
+private List colRefs = ImmutableList.of();
 
 public String getType() {
 return type;
@@ -89,21 +92,36 @@ public class ParameterDesc {
 
 ParameterDesc that = (ParameterDesc) o;
 
-if (nextParameter != null ? !nextParameter.equals(that.nextParameter) 
: that.nextParameter != null)
-return false;
 if (type != null ? !type.equals(that.type) : that.type != null)
 return false;
-if (value != null ? !value.equals(that.value) : that.value != null)
-return false;
-
-return true;
+
+ParameterDesc p = this, q = that;
+int refi = 0, refj = 0;
+for (; p != null && q != null; p = p.nextParameter, q = 
q.nextParameter) {
+if (p.isColumnType()) {
+if (q.isColumnType() == false)
+return false;
+if (refi >= this.colRefs.size() || refj >= that.colRefs.size())
+return false;
+if (this.colRefs.get(refi).equals(that.colRefs.get(refj)) == 
false)
+return false;
+refi++;
+refj++;
+} else {
+if (q.isColumnType() == true)
+return false;
+if (p.value.equals(q.value) == false)
+return false;
+}
+}
+
+return p == null && q == null;
 }
 
 @Override
 public int hashCode() {
 int result = type != null ? type.hashCode() : 0;
-result = 31 * result + (value != null ? value.hashCode() : 0);
-result = 31 * result + (nextParameter != null ? 
nextParameter.hashCode() : 0);
+result = 31 * result + (colRefs != null ? colRefs.hashCode() : 0);
 return result;
 }
 

[09/18] kylin git commit: minor, fix date format in sample data

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/21bcd2f6/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index e13b7ad..215 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,14/12/2012,Others,88750,0,11,36.2828,4,1349,10002313,ANALYST,Beijing
-1,28/08/2012,Others,175750,0,13,23.8563,20,1927,10004376,ANALYST,Beijing
-2,16/02/2012,ABIN,148324,15,13,88.3418,18,1005,10006710,ADMIN,Shanghai
-3,19/10/2013,FP-non GTC,37831,0,13,47.3015,3,1209,10003717,ANALYST,Beijing
-4,22/10/2012,Others,140746,100,11,83.454,16,1154,10006076,ADMIN,Shanghai
-5,25/01/2013,FP-GTC,16509,0,-99,22.9896,11,1372,10007436,ADMIN,Shanghai
-6,04/04/2013,Others,963,0,13,88.5907,11,1648,10009869,MODELER,Hongkong
-7,11/04/2012,Others,15687,0,15,88.194,19,1866,1400,ADMIN,Shanghai
-8,19/01/2013,ABIN,60606,3,13,77.9727,13,1936,1687,MODELER,Hongkong
-9,30/04/2012,FP-non 
GTC,106246,0,14,52.295,17,1047,10009223,MODELER,Hongkong
-10,03/02/2013,Auction,45333,0,16,56.3584,10,1470,10006759,MODELER,Hongkong
-11,02/09/2012,FP-GTC,25147,0,-99,15.3553,8,1203,1062,ANALYST,Beijing
-12,30/07/2013,Others,24760,0,16,25.077,5,1382,10003539,ADMIN,Shanghai
-13,17/04/2012,Auction,31519,0,16,72.7384,11,1069,10001245,ADMIN,Shanghai
-14,30/04/2013,FP-GTC,51582,0,15,75.82,6,1085,10008148,ANALYST,Beijing
-15,23/03/2013,FP-GTC,16509,0,15,8.6653,15,1559,10008079,MODELER,Hongkong
-16,01/04/2013,ABIN,175750,3,16,64.2802,16,1507,10007797,MODELER,Hongkong
-17,12/07/2013,ABIN,50677,0,-99,24.5987,17,1938,10006224,ADMIN,Shanghai
-18,23/04/2012,FP-GTC,45333,0,5,72.6553,15,1863,10003824,MODELER,Hongkong
-19,19/12/2012,ABIN,3838,0,12,87.4142,4,1612,10007157,ADMIN,Shanghai
-20,15/08/2012,FP-GTC,62179,0,16,67.4238,15,1839,10003452,MODELER,Hongkong
-21,22/08/2013,Auction,1357,0,5,18.36,9,1303,1114,ANALYST,Beijing
-22,23/08/2013,ABIN,31519,0,5,67.6344,2,1877,10007417,ANALYST,Beijing
-23,12/09/2013,FP-GTC,223,0,14,71.7898,15,1848,1627,MODELER,Hongkong
-24,14/03/2012,FP-GTC,132939,0,15,26.1496,20,1815,10005292,MODELER,Hongkong
-25,13/07/2013,Others,67703,3,16,83.9661,2,1858,10008123,MODELER,Hongkong
-26,15/12/2013,FP-non GTC,100847,0,5,34.3451,16,1387,10007806,ADMIN,Shanghai
-27,30/08/2012,Auction,62179,0,11,21.0496,9,1393,10007360,ANALYST,Beijing
-28,10/01/2013,FP-GTC,48027,0,13,23.4142,20,1757,10002135,MODELER,Hongkong
-29,25/01/2013,ABIN,145970,0,5,26.7842,6,1402,10009682,ANALYST,Beijing
-30,05/07/2012,Auction,48027,0,-99,46.732,5,1972,10009522,MODELER,Hongkong
-31,07/11/2012,FP-GTC,20213,0,-99,19.727,8,1873,10009944,ANALYST,Beijing
-32,20/06/2013,ABIN,100847,0,5,76.2783,1,1409,10005395,ADMIN,Shanghai
-33,08/10/2012,ABIN,26262,0,5,48.7447,4,1029,10001836,ADMIN,Shanghai
-34,30/04/2013,ABIN,87118,0,15,63.2187,11,1529,1928,ANALYST,Beijing
-35,24/04/2013,FP-non 
GTC,20213,0,16,39.5057,19,1329,1768,ANALYST,Beijing
-36,13/05/2012,Others,164261,0,13,84.1246,15,1688,1727,ADMIN,Shanghai
-37,02/08/2013,ABIN,31519,0,5,37.1504,14,1905,1719,MODELER,Hongkong
-38,03/04/2012,Auction,164261,0,11,92.0974,13,1341,10001337,ADMIN,Shanghai
-39,02/09/2012,FP-GTC,31519,0,14,87.7829,20,1460,10008484,ANALYST,Beijing
-40,05/10/2013,ABIN,13987,0,16,74.0719,4,1381,10002975,MODELER,Hongkong
-41,13/01/2012,FP-GTC,48027,0,15,71.2637,14,1915,10005778,ADMIN,Shanghai
-42,01/03/2013,Others,13836,0,14,16.7288,14,1953,10009385,ADMIN,Shanghai
-43,12/09/2012,FP-non 
GTC,16509,0,5,12.2933,20,1277,10002286,MODELER,Hongkong
-44,07/02/2012,Auction,45333,0,5,64.977,16,1040,10005583,MODELER,Hongkong
-45,14/12/2013,FP-non 
GTC,158798,0,16,72.4413,18,1500,10007635,MODELER,Hongkong
-46,13/10/2013,Auction,31519,0,5,79.3053,17,1816,10006423,MODELER,Hongkong
-47,11/06/2012,ABIN,4943,0,5,11.6942,15,1696,10001237,ANALYST,Beijing
-48,18/10/2012,ABIN,80053,0,-99,54.0933,4,1604,10006851,MODELER,Hongkong
-49,03/11/2012,Auction,20886,0,5,9.8258,8,1764,10001220,MODELER,Hongkong
-50,15/01/2012,Auction,44079,0,14,13.0371,19,1343,10001259,MODELER,Hongkong
-51,03/07/2012,FP-GTC,159184,0,-99,92.5314,4,1724,10009008,ADMIN,Shanghai
-52,16/02/2012,FP-GTC,314,0,13,49.1825,20,1334,10004005,ADMIN,Shanghai
-53,17/06/2012,FP-non GTC,31387,3,12,0.6677,5,1603,10002595,ANALYST,Beijing
-54,15/07/2012,FP-non 
GTC,32876,0,-99,50.9634,11,1248,10006890,ANALYST,Beijing
-55,24/07/2012,FP-GTC,150265,15,16,57.5645,6,1653,10007269,ADMIN,Shanghai
-56,06/10/2012,Auction,145970,0,5,44.3091,16,1082,10001640,ADMIN,Shanghai
-57,14/06/2013,ABIN,80053,0,5,20.2388,19,1632,10002068,MODELER,Hongkong
-58,10/10/2013,FP-non 

[14/18] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

KYLIN-2255 drop v1 coprocessor impl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/545201f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/545201f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/545201f6

Branch: refs/heads/master-hbase1.x
Commit: 545201f6c0c2e4da9b3dcabac0f606e2ef880d19
Parents: 21bcd2f
Author: Li Yang 
Authored: Thu Dec 8 10:41:18 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 10:54:58 2016 +0800

--
 .../apache/kylin/query/ITCombinationTest.java   |   15 -
 .../org/apache/kylin/query/KylinTestBase.java   |2 -
 .../kylin/rest/controller/CubeController.java   |   14 -
 .../hbase/cube/v1/CubeSegmentTupleIterator.java |  344 -
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  796 ---
 .../hbase/cube/v1/CubeTupleConverter.java   |  271 
 .../hbase/cube/v1/RegionScannerAdapter.java |   97 --
 .../hbase/cube/v1/ResultScannerAdapter.java |  100 --
 .../cube/v1/SerializedHBaseTupleIterator.java   |  156 ---
 .../observer/AggregateRegionObserver.java   |  112 --
 .../observer/AggregationScanner.java|  188 ---
 .../observer/ObserverAggregationCache.java  |  166 ---
 .../observer/ObserverAggregators.java   |  265 
 .../coprocessor/observer/ObserverEnabler.java   |  191 ---
 .../v1/coprocessor/observer/ObserverTuple.java  |   71 -
 .../hbase/cube/v1/filter/FuzzyRowFilterV2.java  |  574 
 .../hbase/cube/v1/filter/UnsafeAccess.java  |  433 --
 .../v1/filter/generated/FilterProtosExt.java| 1298 --
 .../cube/v1/filter/protobuf/FilterExt.proto |   39 -
 .../cube/MeasureTypeOnlyAggrInBaseTest.java |   21 -
 .../observer/AggregateRegionObserverTest.java   |  339 -
 .../observer/RowAggregatorsTest.java|   62 -
 .../cube/v1/filter/TestFuzzyRowFilterV2.java|  249 
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  346 -
 24 files changed, 6149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 496ac4e..84573b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.query.routing.Candidate;
 import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.storage.hbase.HBaseStorage;
-import 
org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.runner.RunWith;
@@ -67,7 +66,6 @@ public class ITCombinationTest extends ITKylinQueryTest {
 public static Collection configs() {
 return Arrays.asList(new Object[][] { //
 { "inner", "on", "v2", true }, //
-{ "left", "on", "v1", true }, //
 { "left", "on", "v2", true }, //
 //{ "inner", "on", "v2", false }, // exclude view to simply 
model/cube selection
 //{ "left", "on", "v1", false }, // exclude view to simply 
model/cube selection
@@ -84,23 +82,10 @@ public class ITCombinationTest extends ITKylinQueryTest {
 ITKylinQueryTest.joinType = joinType;
 ITKylinQueryTest.setupAll();
 
-if (coprocessorToggle.equals("on")) {
-ObserverEnabler.forceCoprocessorOn();
-} else if (coprocessorToggle.equals("off")) {
-ObserverEnabler.forceCoprocessorOff();
-} else if (coprocessorToggle.equals("unset")) {
-// unset
-}
-
 RemoveBlackoutRealizationsRule.blackList.clear();
 if (excludeViewCubes) {
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
 }
-
-if ("v1".equalsIgnoreCase(queryEngine))
-HBaseStorage.overwriteStorageQuery = 
HBaseStorage.v1CubeStorageQuery;
-else
-HBaseStorage.overwriteStorageQuery = null;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java 

[03/18] kylin git commit: KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc

[liyang]

KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64d9b8b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64d9b8b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64d9b8b8

Branch: refs/heads/master-hbase1.x
Commit: 64d9b8b89f84cc31f9931e5a6cdf89bdfa13925a
Parents: 227bbf4
Author: Li Yang 
Authored: Thu Dec 1 14:10:44 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:37 2016 +0800

--
 .../apache/kylin/cube/RawQueryLastHacker.java   |  8 +---
 .../metadata/measure/MeasureCodecTest.java  |  3 +-
 .../kylin/measure/raw/RawMeasureType.java   | 12 ++---
 .../kylin/metadata/model/FunctionDesc.java  | 26 ---
 .../kylin/metadata/model/ParameterDesc.java | 47 +++-
 .../apache/kylin/storage/StorageMockUtils.java  | 32 +++--
 .../kylin/query/relnode/OLAPAggregateRel.java   | 12 ++---
 7 files changed, 59 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index b0a4823..682e48c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -57,12 +57,8 @@ public class RawQueryLastHacker {
 sqlDigest.groupbyColumns.add(col);
 } else {
 // For measure columns, take them as metric columns with 
aggregation function SUM().
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc sumFunc = new FunctionDesc();
-sumFunc.setExpression("SUM");
-sumFunc.setParameter(colParameter);
+ParameterDesc parameter = ParameterDesc.newInstance(col);
+FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", 
parameter, null);
 
 boolean measureHasSum = false;
 for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 6bb71e9..18680ec 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -83,8 +83,7 @@ public class MeasureCodecTest extends 
LocalFileMetadataTestCase {
 
 private MeasureDesc measure(String returnType) {
 MeasureDesc desc = new MeasureDesc();
-FunctionDesc func = new FunctionDesc();
-func.setReturnType(returnType);
+FunctionDesc func = FunctionDesc.newInstance(null, null, returnType);
 desc.setFunction(func);
 return desc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 3a49d31..a5bb06b 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -197,18 +197,12 @@ public class RawMeasureType extends 
MeasureType {
 if (sqlDigest.isRawQuery) {
 for (MeasureDesc measureDesc : measureDescs) {
 TblColRef col = this.getRawColumn(measureDesc.getFunction());
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc rawFunc = new FunctionDesc();
-rawFunc.setExpression("RAW");
-rawFunc.setParameter(colParameter);
+ParameterDesc colParameter = ParameterDesc.newInstance(col);
+FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", 
colParameter, null);
 
 if 

[01/18] kylin git commit: KYLIN-1875 Normalize table and column names in JSON [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master-hbase1.x 469d9cc5c -> eaf0cc586 (forced update)


KYLIN-1875 Normalize table and column names in JSON


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/967ef180
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/967ef180
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/967ef180

Branch: refs/heads/master-hbase1.x
Commit: 967ef18062048e199cd0dd351ba82618a10b08e5
Parents: 081ed0d
Author: Li Yang 
Authored: Tue Nov 29 20:32:04 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:35 2016 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   | 22 +---
 .../org/apache/kylin/cube/model/CubeDesc.java   |  1 +
 .../apache/kylin/cube/model/DimensionDesc.java  | 19 +-
 .../model/validation/rule/FunctionRule.java | 37 
 .../kylin/metadata/model/DataModelDesc.java | 13 ++-
 .../kylin/metadata/model/FunctionDesc.java  |  5 +--
 .../metadata/model/ModelDimensionDesc.java  | 19 +++---
 .../kylin/metadata/model/PartitionDesc.java |  4 +--
 8 files changed, 64 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 677b713..011035b 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -81,8 +81,7 @@ public class FactTableGenerator {
 // table(appear as fk in fact table)
 TreeMap lookupTableKeys = new 
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
-// possible values of lookupTableKeys, extracted from existing lookup
-// tables.
+// possible values of lookupTableKeys, extracted from existing lookup 
tables.
 // The key is in the format of tablename/columnname
 TreeMap feasibleValues = new 
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
@@ -244,7 +243,7 @@ public class FactTableGenerator {
 JoinDesc join = dim.getJoin();
 if (join != null) {
 String lookupTable = dim.getTableRef().getTableIdentity();
-for (String column : join.getPrimaryKey()) {
+for (String column : dropAlias(join.getPrimaryKey())) {
 if (!lookupTableKeys.containsKey(lookupTable)) {
 lookupTableKeys.put(lookupTable, new 
LinkedList());
 }
@@ -297,8 +296,8 @@ public class FactTableGenerator {
 for (DimensionDesc dim : dimensions) {
 JoinDesc jDesc = dim.getJoin();
 if (jDesc != null) {
-String[] fks = jDesc.getForeignKey();
-String[] pks = jDesc.getPrimaryKey();
+String[] fks = dropAlias(jDesc.getForeignKey());
+String[] pks = dropAlias(jDesc.getPrimaryKey());
 int num = fks.length;
 for (int i = 0; i < num; ++i) {
 String value = dim.getTableRef().getTableIdentity() + "/" 
+ pks[i];
@@ -343,6 +342,19 @@ public class FactTableGenerator {
 return createTable(this.rowCount, factTableCol2LookupCol, 
lookupCol2factTableCol, usedCols);
 }
 
+private String[] dropAlias(String[] aliasDotCol) {
+String[] result = new String[aliasDotCol.length];
+for (int i = 0; i < aliasDotCol.length; i++) {
+String str = aliasDotCol[i];
+int cut = str.lastIndexOf('.');
+if (cut >= 0) {
+str = str.substring(cut + 1);
+}
+result[i] = str;
+}
+return result;
+}
+
 private String normToTwoDigits(int v) {
 if (v < 10)
 return "0" + v;

http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 853571c..f95cceb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -543,6 +543,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 initMeasureColumns();
 
 rowkey.init(this);
+
 

[17/18] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..8f7430e 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -190,7 +192,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -205,7 +207,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClass)) {
 desc.removeCoprocessor(CubeObserverClass);
@@ -231,16 +233,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -261,12 +262,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch 

[16/18] kylin git commit: KYLIN-2217 fix NPE in FactDistinctColumnsReducer

[liyang]

KYLIN-2217 fix NPE in FactDistinctColumnsReducer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c39fcb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c39fcb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c39fcb0

Branch: refs/heads/master-hbase1.x
Commit: 4c39fcb0eaab053d43554ac6add377343d585065
Parents: c0c56f4
Author: Li Yang 
Authored: Thu Dec 8 11:14:31 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:14:31 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c39fcb0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a3e61a1..776d750 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -117,7 +117,6 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

[07/18] kylin git commit: minor, add API to get AggregationGroup combination cuboid num

[liyang]

minor, add API to get AggregationGroup combination cuboid num


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1fd4679
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1fd4679
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1fd4679

Branch: refs/heads/master-hbase1.x
Commit: b1fd4679fa80e8e971d553a42baf20198afb9eac
Parents: 365caae
Author: Billy Liu 
Authored: Wed Dec 7 18:29:07 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 18:29:07 2016 +0800

--
 .../kylin/rest/controller/CubeController.java   | 33 
 1 file changed, 33 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b1fd4679/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f537231..ab32551 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -655,6 +656,22 @@ public class CubeController extends BasicController {
 return response;
 }
 
+/**
+ * Calculate Cuboid Combination based on the AggreationGroup definition.
+ *
+ * @param aggregationGroupStr
+ * @return number of cuboid, -1 if failed
+ */
+@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.POST)
+@ResponseBody
+public long calculateCuboidCombination(@RequestBody String 
aggregationGroupStr) {
+AggregationGroup aggregationGroup = 
deserializeAggregationGroup(aggregationGroupStr);
+if (aggregationGroup != null) {
+return aggregationGroup.calculateCuboidCombination();
+} else
+return -1;
+}
+
 private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
 CubeDesc desc = null;
 try {
@@ -673,6 +690,22 @@ public class CubeController extends BasicController {
 return desc;
 }
 
+private AggregationGroup deserializeAggregationGroup(String 
aggregationGroupStr) {
+AggregationGroup aggreationGroup = null;
+try {
+logger.debug("Parsing AggregationGroup " + aggregationGroupStr);
+aggreationGroup = JsonUtil.readValue(aggregationGroupStr, 
AggregationGroup.class);
+} catch (JsonParseException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (JsonMappingException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (IOException e) {
+logger.error("Failed to deal with the request.", e);
+throw new InternalErrorException("Failed to deal with the 
request:" + e.getMessage(), e);
+}
+return aggreationGroup;
+}
+
 private void updateRequest(CubeRequest request, boolean success, String 
message) {
 request.setCubeDescData("");
 request.setSuccessful(success);

[05/18] kylin git commit: KYLIN-1875 fix PK-FK derive from each other

[liyang]

KYLIN-1875 fix PK-FK derive from each other


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92e4d464
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92e4d464
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92e4d464

Branch: refs/heads/master-hbase1.x
Commit: 92e4d464069745f0c58cc0559e054efa470960c1
Parents: 79e4801
Author: Yang Li 
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:39 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  8 +--
 .../org/apache/kylin/cube/model/CubeDesc.java   | 52 +++-
 .../storage/gtrecord/CubeTupleConverter.java|  2 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  2 +-
 .../translate/DerivedFilterTranslator.java  |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../query/enumerator/LookupTableEnumerator.java |  2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  2 +-
 .../hbase/cube/v1/CubeTupleConverter.java   |  2 +-
 9 files changed, 41 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index bda1423..fe2030a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -44,7 +44,6 @@ import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -54,6 +53,7 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -629,10 +629,10 @@ public class CubeManager implements IRealizationProvider {
 cubeMap.removeLocal(cubeName);
 }
 
-public LookupStringTable getLookupTable(CubeSegment cubeSegment, 
DimensionDesc dim) {
+public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc 
join) {
 
-String tableName = dim.getTableRef().getTableIdentity();
-String[] pkCols = dim.getJoin().getPrimaryKey();
+String tableName = join.getPKSide().getTableIdentity();
+String[] pkCols = join.getPrimaryKey();
 String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
 if (snapshotResPath == null)
 throw new IllegalStateException("No snaphot for table '" + 
tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() 
+ "/" + cubeSegment);

http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f95cceb..f8c316c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -101,20 +102,20 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public static class DeriveInfo {
 public DeriveType type;
-public DimensionDesc dimension;
+public JoinDesc join;
 public TblColRef[] columns;
 public boolean isOneToOne; // only used when ref from derived to host
 
-DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] 
columns, boolean isOneToOne) {
+DeriveInfo(DeriveType type, JoinDesc 

[08/18] kylin git commit: KYLIN-2254 allow 2 columns in a CompareTupleFilter

[liyang]

KYLIN-2254 allow 2 columns in a CompareTupleFilter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/350fad8f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/350fad8f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/350fad8f

Branch: refs/heads/master-hbase1.x
Commit: 350fad8f248988596c05b080b3c5a68b77b790a1
Parents: b1fd467
Author: Li Yang 
Authored: Wed Dec 7 17:51:26 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 09:47:27 2016 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 23 ++--
 .../kylin/metadata/model/FunctionDesc.java  |  2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |  2 +-
 .../resources/query/sql_subquery/query12.sql| 37 
 .../kylin/query/relnode/OLAPAggregateRel.java   | 13 ---
 .../kylin/query/relnode/OLAPFilterRel.java  | 30 +---
 6 files changed, 66 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..d567c89 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -37,6 +37,7 @@ public class CompareTupleFilter extends TupleFilter {
 // operand 1 is either a column or a function
 private TblColRef column;
 private FunctionTupleFilter function;
+private TblColRef secondColumn;
 
 // operand 2 is constants
 private Set conditionValues;
@@ -73,15 +74,16 @@ public class CompareTupleFilter extends TupleFilter {
 if (child instanceof ColumnTupleFilter) {
 ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
 if (this.column != null) {
-throw new IllegalStateException("Duplicate columns! old is " + 
column.getName() + " and new is " + columnFilter.getColumn().getName());
-}
-this.column = columnFilter.getColumn();
-// if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
-// children.size() > 1 means already added one conditionValue or 
dynamicVariable
-if (this.children.size() > 1 && needSwapOperator()) {
-this.operator = SWAP_OP_MAP.get(this.operator);
-TupleFilter last = this.children.remove(this.children.size() - 
1);
-this.children.add(0, last);
+this.secondColumn = columnFilter.getColumn();
+} else {
+this.column = columnFilter.getColumn();
+// if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
+// children.size() > 1 means already added one conditionValue 
or dynamicVariable
+if (this.children.size() > 1 && needSwapOperator()) {
+this.operator = SWAP_OP_MAP.get(this.operator);
+TupleFilter last = 
this.children.remove(this.children.size() - 1);
+this.children.add(0, last);
+}
 }
 } else if (child instanceof ConstantTupleFilter) {
 this.conditionValues.addAll(child.getValues());
@@ -212,7 +214,8 @@ public class CompareTupleFilter extends TupleFilter {
 
 @Override
 public boolean isEvaluable() {
-return ((function != null && function.isEvaluable()) || column != 
null) && !conditionValues.isEmpty();
+return (column != null || (function != null && 
function.isEvaluable())) //
+&& !conditionValues.isEmpty() && secondColumn == null;
 }
 
 @SuppressWarnings({ "unchecked", "rawtypes" })

http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 832cb4a..9252c42 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -140,7 +140,7 @@ public class FunctionDesc {
 } else if (isCount()) {
 return "COUNT__"; // ignores parameter, count(*), count(1), 
count(col) are all the 

[15/18] kylin git commit: KYLIN-2217 fix null partition col bug

[liyang]

KYLIN-2217 fix null partition col bug

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c56f4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c56f4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c56f4b

Branch: refs/heads/master-hbase1.x
Commit: c0c56f4b6b05bb108a4952a56af992b237c1bbe6
Parents: 545201f
Author: xiefan46 <958034...@qq.com>
Authored: Thu Dec 8 10:33:48 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:09:31 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java| 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c56f4b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8933ee2..a3e61a1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -116,13 +116,17 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

[13/18] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
deleted file mode 100644
index a900ea1..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.gridtable.StorageSideBehavior;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-
-/**
- * @author yangli9
- * 
- */
-public class AggregationScanner implements RegionScanner {
-
-private RegionScanner outerScanner;
-private StorageSideBehavior behavior;
-
-public AggregationScanner(CoprocessorRowType type, CoprocessorFilter 
filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner 
innerScanner, StorageSideBehavior behavior) throws IOException {
-
-AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
-
-this.behavior = behavior;
-
-ObserverAggregationCache aggCache;
-Stats stats = new Stats();
-
-aggCache = buildAggrCache(innerScanner, type, groupBy, aggrs, filter, 
stats);
-stats.countOutputRow(aggCache.getSize());
-this.outerScanner = aggCache.getScanner(innerScanner);
-
-AggregateRegionObserver.LOG.info("Kylin Coprocessor aggregation done: 
" + stats);
-}
-
-@SuppressWarnings("rawtypes")
-ObserverAggregationCache buildAggrCache(final RegionScanner innerScanner, 
CoprocessorRowType type, CoprocessorProjector projector, ObserverAggregators 
aggregators, CoprocessorFilter filter, Stats stats) throws IOException {
-
-ObserverAggregationCache aggCache = new 
ObserverAggregationCache(aggregators);
-
-ObserverTuple tuple = new ObserverTuple(type);
-boolean hasMore = true;
-List results = new ArrayList();
-byte meaninglessByte = 0;
-
-while (hasMore) {
-results.clear();
-hasMore = innerScanner.nextRaw(results);
-if (results.isEmpty())
-continue;
-
-if (stats != null)
-stats.countInputRow(results);
-
-Cell cell = results.get(0);
-tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), 
cell.getRowLength());
-
-if (behavior == StorageSideBehavior.SCAN) {
-//touch every byte of the cell so that the cost of scanning 
will be trully reflected
-int endIndex = cell.getRowOffset() + cell.getRowLength();
-for (int i = cell.getRowOffset(); i < endIndex; ++i) {
-meaninglessByte += cell.getRowArray()[i];
-}
-} else {
-if (behavior.filterToggledOn()) {
-if (filter != null && filter.evaluate(tuple) == false)
-continue;
-
-if (behavior.aggrToggledOn()) {
-AggrKey aggKey = projector.getAggrKey(results);
-MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
-aggregators.aggregate(bufs, results);
-
-if (behavior.ordinal() >= 

[18/18] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eaf0cc58
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eaf0cc58
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eaf0cc58

Branch: refs/heads/master-hbase1.x
Commit: eaf0cc5866367eadacfb7a43ca793c2f585dfdc6
Parents: 4c39fcb
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 13:42:27 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 40 files changed, 365 insertions(+), 432 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 

[12/18] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
deleted file mode 100644
index 1bd92e6..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+++ /dev/null
@@ -1,1298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: 
storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
-
-package org.apache.kylin.storage.hbase.cube.v1.filter.generated;
-
-public final class FilterProtosExt {
-private FilterProtosExt() {
-}
-
-public static void 
registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
-}
-
-public interface BytesBytesPairOrBuilder extends 
com.google.protobuf.MessageOrBuilder {
-
-// required bytes first = 1;
-/**
- * required bytes first = 1;
- */
-boolean hasFirst();
-
-/**
- * required bytes first = 1;
- */
-com.google.protobuf.ByteString getFirst();
-
-// required bytes second = 2;
-/**
- * required bytes second = 2;
- */
-boolean hasSecond();
-
-/**
- * required bytes second = 2;
- */
-com.google.protobuf.ByteString getSecond();
-}
-
-/**
- * Protobuf type {@code BytesBytesPair}
- */
-public static final class BytesBytesPair extends 
com.google.protobuf.GeneratedMessage implements BytesBytesPairOrBuilder {
-// Use BytesBytesPair.newBuilder() to construct.
-private BytesBytesPair(com.google.protobuf.GeneratedMessage.Builder 
builder) {
-super(builder);
-this.unknownFields = builder.getUnknownFields();
-}
-
-private BytesBytesPair(boolean noInit) {
-this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance();
-}
-
-private static final BytesBytesPair defaultInstance;
-
-public static BytesBytesPair getDefaultInstance() {
-return defaultInstance;
-}
-
-public BytesBytesPair getDefaultInstanceForType() {
-return defaultInstance;
-}
-
-private final com.google.protobuf.UnknownFieldSet unknownFields;
-
-@java.lang.Override
-public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
-return this.unknownFields;
-}
-
-private BytesBytesPair(com.google.protobuf.CodedInputStream input, 
com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws 
com.google.protobuf.InvalidProtocolBufferException {
-initFields();
-int mutable_bitField0_ = 0;
-com.google.protobuf.UnknownFieldSet.Builder unknownFields = 
com.google.protobuf.UnknownFieldSet.newBuilder();
-try {
-boolean done = false;
-while (!done) {
-int tag = input.readTag();
-switch (tag) {
-case 0:
-done = true;
-break;
-default: {
-if (!parseUnknownField(input, unknownFields, 
extensionRegistry, tag)) {
-done = true;
-}
-break;
-}
-case 10: {
-bitField0_ |= 0x0001;
-first_ = input.readBytes();
-break;
-}
-case 18: {
-bitField0_ |= 0x0002;
-second_ = input.readBytes();
-break;
-}
-}
-}
-

[06/18] kylin git commit: KYLIN-1875 minor, test fixes

[liyang]

KYLIN-1875 minor, test fixes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/365caaeb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/365caaeb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/365caaeb

Branch: refs/heads/master-hbase1.x
Commit: 365caaebd841917f161dab37b0af7c055626194c
Parents: 92e4d46
Author: Yang Li 
Authored: Fri Dec 2 06:32:25 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:46 2016 +0800

--
 .../metadata/filter/ColumnTupleFilter.java  |  1 +
 .../kylin/metadata/model/DataModelDesc.java |  2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |  2 +-
 .../template/model_desc/kylin_sales_model.json  |  4 +++
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../resources/query/sql_subquery/query02.sql| 31 
 .../query/sql_subquery/query02.sql.disable  | 25 
 7 files changed, 39 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index c8a8f07..ecb8e61 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -121,6 +121,7 @@ public class ColumnTupleFilter extends TupleFilter {
 }
 }
 
+@SuppressWarnings("deprecation")
 @Override
 public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 898ff74..3f868a2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -112,7 +112,7 @@ public class DataModelDesc extends RootPersistentEntity {
 return name;
 }
 
-// for test only
+// for test mainly
 @Deprecated
 public void setName(String name) {
 this.name = name;

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index bf8d36b..5d72c3f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -84,7 +84,7 @@ public class TblColRef implements Serializable {
 col.table = tableRef;
 }
 
-// for test only
+// for test mainly
 public static TblColRef mockup(TableDesc table, int oneBasedColumnIndex, 
String name, String datatype) {
 ColumnDesc desc = new ColumnDesc();
 String id = "" + oneBasedColumnIndex;

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 2d3cfb6..cce360f 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -18,6 +18,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_ACCOUNT",
 "alias" : "BUYER_ACCOUNT",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "ACCOUNT_ID" ],
@@ -26,6 +27,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_ACCOUNT",
 "alias" : "SELLER_ACCOUNT",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "ACCOUNT_ID" ],
@@ -34,6 +36,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_COUNTRY",
 "alias" : "BUYER_COUNTRY",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "COUNTRY" ],
@@ -42,6 +45,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_COUNTRY",
 "alias" : "SELLER_COUNTRY",
+"kind" : "LOOKUP",
  

[10/18] kylin git commit: minor, fix date format in sample data

[liyang]

minor, fix date format in sample data


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21bcd2f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21bcd2f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21bcd2f6

Branch: refs/heads/master-hbase1.x
Commit: 21bcd2f6b7d56e639aba18f3f376875922c9cd68
Parents: 350fad8
Author: lidongsjtu 
Authored: Thu Dec 8 10:40:45 2016 +0800
Committer: lidongsjtu 
Committed: Thu Dec 8 10:40:55 2016 +0800

--
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 1 file changed, 1 insertions(+), 1 deletions(-)
--

[04/18] kylin git commit: KYLIN-1875 Support measure on non-root-fact-table

[liyang]

KYLIN-1875 Support measure on non-root-fact-table


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79e48011
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79e48011
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79e48011

Branch: refs/heads/master-hbase1.x
Commit: 79e48011bd3f4696a916f0bfee244ec28355f3b3
Parents: 64d9b8b
Author: Li Yang 
Authored: Thu Dec 1 18:59:08 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:38 2016 +0800

--
 .../kylin/metadata/project/ProjectL2Cache.java  | 26 +---
 .../template/cube_desc/kylin_sales_cube.json| 18 +++---
 .../kylin/query/relnode/OLAPTableScan.java  |  4 +++
 3 files changed, 30 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 82c0de3..14d7843 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
@@ -126,17 +127,17 @@ class ProjectL2Cache {
 return Collections.unmodifiableSet(tableCache.realizations);
 }
 
-public List listEffectiveRewriteMeasures(String project, 
String factTable, boolean onlyRewriteMeasure) {
-Set realizations = getRealizationsByTable(project, 
factTable);
+public List listEffectiveRewriteMeasures(String project, 
String table, boolean onlyRewriteMeasure) {
+Set realizations = getRealizationsByTable(project, 
table);
 List result = Lists.newArrayList();
 for (IRealization r : realizations) {
-if (r.getModel().isFactTable(factTable) && r.isReady()) {
-for (MeasureDesc m : r.getMeasures()) {
-FunctionDesc func = m.getFunction();
-if (onlyRewriteMeasure) {
-if (func.needRewrite())
-result.add(m);
-} else {
+if (!r.isReady())
+continue;
+
+for (MeasureDesc m : r.getMeasures()) {
+FunctionDesc func = m.getFunction();
+if (belongToTable(func, table, r.getModel())) {
+if (!onlyRewriteMeasure || func.needRewrite()) {
 result.add(m);
 }
 }
@@ -145,6 +146,13 @@ class ProjectL2Cache {
 return result;
 }
 
+private boolean belongToTable(FunctionDesc func, String table, 
DataModelDesc model) {
+// measure belong to the first column parameter's table
+List cols = func.getParameter().getColRefs();
+String belongTo = cols.isEmpty() ? 
model.getRootFactTable().getTableIdentity() : cols.get(0).getTable();
+return belongTo.equals(table);
+}
+
 // 

 // build the cache
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 48eef46..2a27305 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -103,26 +103,26 @@
   "returntype" : "decimal(19,4)"
 }
   }, {
-"name" : "GMV_MIN",
+"name" : "BUYER_LEVEL_SUM",
 "function" : {
-  "expression" : "MIN",
+  "expression" : "SUM",
   "parameter" : {
 "type" : "column",
-"value" : "PRICE",
+"value" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
 "next_parameter" : null
   },
-  "returntype" : "decimal(19,4)"
+  "returntype" : "bigint"
 }
   }, {
-"name" : "GMV_MAX",
+"name" : "SELLER_LEVEL_SUM",
 "function" : {
-  "expression" : "MAX",
+  

[02/18] kylin git commit: KYLIN-1875 fix ParameterDesc equals() & hashcode()

[liyang]

KYLIN-1875 fix ParameterDesc equals() & hashcode()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/227bbf41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/227bbf41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/227bbf41

Branch: refs/heads/master-hbase1.x
Commit: 227bbf4152074c0621bf2b1047102d416d4dd3eb
Parents: 967ef18
Author: Yang Li 
Authored: Thu Dec 1 06:58:08 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:36 2016 +0800

--
 .../kylin/metadata/model/FunctionDesc.java  |  1 +
 .../kylin/metadata/model/ParameterDesc.java | 36 +++-
 2 files changed, 28 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/227bbf41/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index dfa6f3b..b9e5543 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -248,6 +248,7 @@ public class FunctionDesc {
 final int prime = 31;
 int result = 1;
 result = prime * result + ((expression == null) ? 0 : 
expression.hashCode());
+result = prime * result + ((returnType == null) ? 0 : 
returnType.hashCode());
 result = prime * result + ((isCount() || parameter == null) ? 0 : 
parameter.hashCode());
 return result;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/227bbf41/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 329799f..4a95fea 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -22,8 +22,10 @@ import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableList;
 
 /**
  */
@@ -36,9 +38,10 @@ public class ParameterDesc {
 private String value;
 
 @JsonProperty("next_parameter")
+@JsonInclude(JsonInclude.Include.NON_NULL)
 private ParameterDesc nextParameter;
 
-private List colRefs;
+private List colRefs = ImmutableList.of();
 
 public String getType() {
 return type;
@@ -89,21 +92,36 @@ public class ParameterDesc {
 
 ParameterDesc that = (ParameterDesc) o;
 
-if (nextParameter != null ? !nextParameter.equals(that.nextParameter) 
: that.nextParameter != null)
-return false;
 if (type != null ? !type.equals(that.type) : that.type != null)
 return false;
-if (value != null ? !value.equals(that.value) : that.value != null)
-return false;
-
-return true;
+
+ParameterDesc p = this, q = that;
+int refi = 0, refj = 0;
+for (; p != null && q != null; p = p.nextParameter, q = 
q.nextParameter) {
+if (p.isColumnType()) {
+if (q.isColumnType() == false)
+return false;
+if (refi >= this.colRefs.size() || refj >= that.colRefs.size())
+return false;
+if (this.colRefs.get(refi).equals(that.colRefs.get(refj)) == 
false)
+return false;
+refi++;
+refj++;
+} else {
+if (q.isColumnType() == true)
+return false;
+if (p.value.equals(q.value) == false)
+return false;
+}
+}
+
+return p == null && q == null;
 }
 
 @Override
 public int hashCode() {
 int result = type != null ? type.hashCode() : 0;
-result = 31 * result + (value != null ? value.hashCode() : 0);
-result = 31 * result + (nextParameter != null ? 
nextParameter.hashCode() : 0);
+result = 31 * result + (colRefs != null ? colRefs.hashCode() : 0);
 return result;
 }
 

[2/5] kylin git commit: KYLIN-2217 fix NPE in FactDistinctColumnsReducer

[liyang]

KYLIN-2217 fix NPE in FactDistinctColumnsReducer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac621fea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac621fea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac621fea

Branch: refs/heads/yang22-cdh5.7
Commit: ac621fea49fa220a3d14dc4600bcbd3961e7580f
Parents: ae31b13
Author: Li Yang 
Authored: Thu Dec 8 11:14:31 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:15:15 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac621fea/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a3e61a1..776d750 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -117,7 +117,6 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

[1/5] kylin git commit: KYLIN-2217 fix null partition col bug [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 6144e8ec0 -> 4ae2df2d3 (forced update)


KYLIN-2217 fix null partition col bug

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae31b13e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae31b13e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae31b13e

Branch: refs/heads/yang22-cdh5.7
Commit: ae31b13e70ab214a3b5c320f3c2094e1e8c022ef
Parents: 43805bc
Author: xiefan46 <958034...@qq.com>
Authored: Thu Dec 8 10:33:48 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:10:49 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java| 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ae31b13e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8933ee2..a3e61a1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -116,13 +116,17 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

[5/5] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ae2df2d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ae2df2d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ae2df2d

Branch: refs/heads/yang22-cdh5.7
Commit: 4ae2df2d3d2189f1f47c75c7973ff2503075dd88
Parents: 59b85b5
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Li Yang 
Committed: Thu Dec 8 11:40:36 2016 +

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae2df2d/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae2df2d/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae2df2d/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/59b85b5d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/59b85b5d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59b85b5d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59b85b5d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59b85b5d

Branch: refs/heads/yang22-cdh5.7
Commit: 59b85b5d3cca92dc8d7b66815649b765da5794d9
Parents: ac621fe
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:36:50 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59b85b5d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59b85b5d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59b85b5d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59b85b5d

Branch: refs/heads/yang22-hbase1.x
Commit: 59b85b5d3cca92dc8d7b66815649b765da5794d9
Parents: ac621fe
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:36:50 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59b85b5d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/4] kylin git commit: KYLIN-2217 fix null partition col bug [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x e79872182 -> 59b85b5d3 (forced update)


KYLIN-2217 fix null partition col bug

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae31b13e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae31b13e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae31b13e

Branch: refs/heads/yang22-hbase1.x
Commit: ae31b13e70ab214a3b5c320f3c2094e1e8c022ef
Parents: 43805bc
Author: xiefan46 <958034...@qq.com>
Authored: Thu Dec 8 10:33:48 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:10:49 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java| 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ae31b13e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8933ee2..a3e61a1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -116,13 +116,17 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/59b85b5d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/59b85b5d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[2/4] kylin git commit: KYLIN-2217 fix NPE in FactDistinctColumnsReducer

[liyang]

KYLIN-2217 fix NPE in FactDistinctColumnsReducer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac621fea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac621fea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac621fea

Branch: refs/heads/yang22-hbase1.x
Commit: ac621fea49fa220a3d14dc4600bcbd3961e7580f
Parents: ae31b13
Author: Li Yang 
Authored: Thu Dec 8 11:14:31 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:15:15 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac621fea/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a3e61a1..776d750 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -117,7 +117,6 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

kylin git commit: KYLIN-2217 fix NPE in FactDistinctColumnsReducer

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22 ae31b13e7 -> ac621fea4


KYLIN-2217 fix NPE in FactDistinctColumnsReducer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac621fea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac621fea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac621fea

Branch: refs/heads/yang22
Commit: ac621fea49fa220a3d14dc4600bcbd3961e7580f
Parents: ae31b13
Author: Li Yang 
Authored: Thu Dec 8 11:14:31 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:15:15 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac621fea/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a3e61a1..776d750 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -117,7 +117,6 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

kylin git commit: KYLIN-2217 fix NPE in FactDistinctColumnsReducer

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master c0c56f4b6 -> 4c39fcb0e


KYLIN-2217 fix NPE in FactDistinctColumnsReducer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c39fcb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c39fcb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c39fcb0

Branch: refs/heads/master
Commit: 4c39fcb0eaab053d43554ac6add377343d585065
Parents: c0c56f4
Author: Li Yang 
Authored: Thu Dec 8 11:14:31 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:14:31 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c39fcb0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a3e61a1..776d750 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -117,7 +117,6 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

kylin git commit: KYLIN-2217 fix null partition col bug

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22 43805bcd2 -> ae31b13e7


KYLIN-2217 fix null partition col bug

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae31b13e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae31b13e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae31b13e

Branch: refs/heads/yang22
Commit: ae31b13e70ab214a3b5c320f3c2094e1e8c022ef
Parents: 43805bc
Author: xiefan46 <958034...@qq.com>
Authored: Thu Dec 8 10:33:48 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:10:49 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java| 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ae31b13e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8933ee2..a3e61a1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -116,13 +116,17 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

kylin git commit: KYLIN-2217 fix null partition col bug

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 545201f6c -> c0c56f4b6


KYLIN-2217 fix null partition col bug

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c56f4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c56f4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c56f4b

Branch: refs/heads/master
Commit: c0c56f4b6b05bb108a4952a56af992b237c1bbe6
Parents: 545201f
Author: xiefan46 <958034...@qq.com>
Authored: Thu Dec 8 10:33:48 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 11:09:31 2016 +0800

--
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java| 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c56f4b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8933ee2..a3e61a1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -116,13 +116,17 @@ public class FactDistinctColumnsReducer extends 
KylinReducer

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2255 [deleted] 021a42742

[3/4] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
deleted file mode 100644
index a900ea1..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.gridtable.StorageSideBehavior;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-
-/**
- * @author yangli9
- * 
- */
-public class AggregationScanner implements RegionScanner {
-
-private RegionScanner outerScanner;
-private StorageSideBehavior behavior;
-
-public AggregationScanner(CoprocessorRowType type, CoprocessorFilter 
filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner 
innerScanner, StorageSideBehavior behavior) throws IOException {
-
-AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
-
-this.behavior = behavior;
-
-ObserverAggregationCache aggCache;
-Stats stats = new Stats();
-
-aggCache = buildAggrCache(innerScanner, type, groupBy, aggrs, filter, 
stats);
-stats.countOutputRow(aggCache.getSize());
-this.outerScanner = aggCache.getScanner(innerScanner);
-
-AggregateRegionObserver.LOG.info("Kylin Coprocessor aggregation done: 
" + stats);
-}
-
-@SuppressWarnings("rawtypes")
-ObserverAggregationCache buildAggrCache(final RegionScanner innerScanner, 
CoprocessorRowType type, CoprocessorProjector projector, ObserverAggregators 
aggregators, CoprocessorFilter filter, Stats stats) throws IOException {
-
-ObserverAggregationCache aggCache = new 
ObserverAggregationCache(aggregators);
-
-ObserverTuple tuple = new ObserverTuple(type);
-boolean hasMore = true;
-List results = new ArrayList();
-byte meaninglessByte = 0;
-
-while (hasMore) {
-results.clear();
-hasMore = innerScanner.nextRaw(results);
-if (results.isEmpty())
-continue;
-
-if (stats != null)
-stats.countInputRow(results);
-
-Cell cell = results.get(0);
-tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), 
cell.getRowLength());
-
-if (behavior == StorageSideBehavior.SCAN) {
-//touch every byte of the cell so that the cost of scanning 
will be trully reflected
-int endIndex = cell.getRowOffset() + cell.getRowLength();
-for (int i = cell.getRowOffset(); i < endIndex; ++i) {
-meaninglessByte += cell.getRowArray()[i];
-}
-} else {
-if (behavior.filterToggledOn()) {
-if (filter != null && filter.evaluate(tuple) == false)
-continue;
-
-if (behavior.aggrToggledOn()) {
-AggrKey aggKey = projector.getAggrKey(results);
-MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
-aggregators.aggregate(bufs, results);
-
-if (behavior.ordinal() >= 

[2/4] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
deleted file mode 100644
index 1bd92e6..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+++ /dev/null
@@ -1,1298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: 
storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
-
-package org.apache.kylin.storage.hbase.cube.v1.filter.generated;
-
-public final class FilterProtosExt {
-private FilterProtosExt() {
-}
-
-public static void 
registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
-}
-
-public interface BytesBytesPairOrBuilder extends 
com.google.protobuf.MessageOrBuilder {
-
-// required bytes first = 1;
-/**
- * required bytes first = 1;
- */
-boolean hasFirst();
-
-/**
- * required bytes first = 1;
- */
-com.google.protobuf.ByteString getFirst();
-
-// required bytes second = 2;
-/**
- * required bytes second = 2;
- */
-boolean hasSecond();
-
-/**
- * required bytes second = 2;
- */
-com.google.protobuf.ByteString getSecond();
-}
-
-/**
- * Protobuf type {@code BytesBytesPair}
- */
-public static final class BytesBytesPair extends 
com.google.protobuf.GeneratedMessage implements BytesBytesPairOrBuilder {
-// Use BytesBytesPair.newBuilder() to construct.
-private BytesBytesPair(com.google.protobuf.GeneratedMessage.Builder 
builder) {
-super(builder);
-this.unknownFields = builder.getUnknownFields();
-}
-
-private BytesBytesPair(boolean noInit) {
-this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance();
-}
-
-private static final BytesBytesPair defaultInstance;
-
-public static BytesBytesPair getDefaultInstance() {
-return defaultInstance;
-}
-
-public BytesBytesPair getDefaultInstanceForType() {
-return defaultInstance;
-}
-
-private final com.google.protobuf.UnknownFieldSet unknownFields;
-
-@java.lang.Override
-public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
-return this.unknownFields;
-}
-
-private BytesBytesPair(com.google.protobuf.CodedInputStream input, 
com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws 
com.google.protobuf.InvalidProtocolBufferException {
-initFields();
-int mutable_bitField0_ = 0;
-com.google.protobuf.UnknownFieldSet.Builder unknownFields = 
com.google.protobuf.UnknownFieldSet.newBuilder();
-try {
-boolean done = false;
-while (!done) {
-int tag = input.readTag();
-switch (tag) {
-case 0:
-done = true;
-break;
-default: {
-if (!parseUnknownField(input, unknownFields, 
extensionRegistry, tag)) {
-done = true;
-}
-break;
-}
-case 10: {
-bitField0_ |= 0x0001;
-first_ = input.readBytes();
-break;
-}
-case 18: {
-bitField0_ |= 0x0002;
-second_ = input.readBytes();
-break;
-}
-}
-}
-

[1/4] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 21bcd2f6b -> 545201f6c


http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
deleted file mode 100644
index 74b2112..000
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1.filter;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestFuzzyRowFilterV2 {
-@Test
-public void testSatisfiesNoUnsafeForward() {
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, (byte) -128, 1, 0, 1 
}, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, (byte) -128, 2, 0, 1 
}, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 3, 3 }, 0, 5, 
new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
-FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 
1, 3, 0 }, // row to check
-0, 5, new byte[] { 1, 2, 0, 3 }, // fuzzy row
-new byte[] { 0, 0, 1, 0 })); // mask
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, 0, 5, 
new byte[] { 1, (byte) 245, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 0, 1 }, 0, 5, 
new byte[] { 0, 1, 2 }, new byte[] { 1, 0, 0 }));
-}
-
-@Test
-public void testSatisfiesForward() {
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, (byte) -128, 1, 0, 1 }, new 
byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, (byte) -128, 2, 0, 1 }, new 
byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 3, 3 }, new byte[] { 1, 
2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
-FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 
}, // row to check
-new byte[] { 1, 2, 0, 3 }, // fuzzy row
-new byte[] { -1, -1, 0, -1 })); // mask
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, new byte[] { 1, 
(byte) 245, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 0, 1 }, new byte[] { 0, 
1, 2 }, new byte[] { 0, -1, -1 }));
-}
-
-@Test
-public void testSatisfiesReverse() {
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, 
FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) -128, 1, 0, 1 }, new 
byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
-Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, 
FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) -128, 2, 0, 1 }, new 
byte[] { 1, 0, 1 }, new byte[] { 

[4/4] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

KYLIN-2255 drop v1 coprocessor impl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/545201f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/545201f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/545201f6

Branch: refs/heads/master
Commit: 545201f6c0c2e4da9b3dcabac0f606e2ef880d19
Parents: 21bcd2f
Author: Li Yang 
Authored: Thu Dec 8 10:41:18 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 10:54:58 2016 +0800

--
 .../apache/kylin/query/ITCombinationTest.java   |   15 -
 .../org/apache/kylin/query/KylinTestBase.java   |2 -
 .../kylin/rest/controller/CubeController.java   |   14 -
 .../hbase/cube/v1/CubeSegmentTupleIterator.java |  344 -
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  796 ---
 .../hbase/cube/v1/CubeTupleConverter.java   |  271 
 .../hbase/cube/v1/RegionScannerAdapter.java |   97 --
 .../hbase/cube/v1/ResultScannerAdapter.java |  100 --
 .../cube/v1/SerializedHBaseTupleIterator.java   |  156 ---
 .../observer/AggregateRegionObserver.java   |  112 --
 .../observer/AggregationScanner.java|  188 ---
 .../observer/ObserverAggregationCache.java  |  166 ---
 .../observer/ObserverAggregators.java   |  265 
 .../coprocessor/observer/ObserverEnabler.java   |  191 ---
 .../v1/coprocessor/observer/ObserverTuple.java  |   71 -
 .../hbase/cube/v1/filter/FuzzyRowFilterV2.java  |  574 
 .../hbase/cube/v1/filter/UnsafeAccess.java  |  433 --
 .../v1/filter/generated/FilterProtosExt.java| 1298 --
 .../cube/v1/filter/protobuf/FilterExt.proto |   39 -
 .../cube/MeasureTypeOnlyAggrInBaseTest.java |   21 -
 .../observer/AggregateRegionObserverTest.java   |  339 -
 .../observer/RowAggregatorsTest.java|   62 -
 .../cube/v1/filter/TestFuzzyRowFilterV2.java|  249 
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  346 -
 24 files changed, 6149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 496ac4e..84573b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.query.routing.Candidate;
 import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.storage.hbase.HBaseStorage;
-import 
org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.runner.RunWith;
@@ -67,7 +66,6 @@ public class ITCombinationTest extends ITKylinQueryTest {
 public static Collection configs() {
 return Arrays.asList(new Object[][] { //
 { "inner", "on", "v2", true }, //
-{ "left", "on", "v1", true }, //
 { "left", "on", "v2", true }, //
 //{ "inner", "on", "v2", false }, // exclude view to simply 
model/cube selection
 //{ "left", "on", "v1", false }, // exclude view to simply 
model/cube selection
@@ -84,23 +82,10 @@ public class ITCombinationTest extends ITKylinQueryTest {
 ITKylinQueryTest.joinType = joinType;
 ITKylinQueryTest.setupAll();
 
-if (coprocessorToggle.equals("on")) {
-ObserverEnabler.forceCoprocessorOn();
-} else if (coprocessorToggle.equals("off")) {
-ObserverEnabler.forceCoprocessorOff();
-} else if (coprocessorToggle.equals("unset")) {
-// unset
-}
-
 RemoveBlackoutRealizationsRule.blackList.clear();
 if (excludeViewCubes) {
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
 }
-
-if ("v1".equalsIgnoreCase(queryEngine))
-HBaseStorage.overwriteStorageQuery = 
HBaseStorage.v1CubeStorageQuery;
-else
-HBaseStorage.overwriteStorageQuery = null;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java 

[2/4] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/021a4274/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
deleted file mode 100644
index 1bd92e6..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+++ /dev/null
@@ -1,1298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: 
storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
-
-package org.apache.kylin.storage.hbase.cube.v1.filter.generated;
-
-public final class FilterProtosExt {
-private FilterProtosExt() {
-}
-
-public static void 
registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
-}
-
-public interface BytesBytesPairOrBuilder extends 
com.google.protobuf.MessageOrBuilder {
-
-// required bytes first = 1;
-/**
- * required bytes first = 1;
- */
-boolean hasFirst();
-
-/**
- * required bytes first = 1;
- */
-com.google.protobuf.ByteString getFirst();
-
-// required bytes second = 2;
-/**
- * required bytes second = 2;
- */
-boolean hasSecond();
-
-/**
- * required bytes second = 2;
- */
-com.google.protobuf.ByteString getSecond();
-}
-
-/**
- * Protobuf type {@code BytesBytesPair}
- */
-public static final class BytesBytesPair extends 
com.google.protobuf.GeneratedMessage implements BytesBytesPairOrBuilder {
-// Use BytesBytesPair.newBuilder() to construct.
-private BytesBytesPair(com.google.protobuf.GeneratedMessage.Builder 
builder) {
-super(builder);
-this.unknownFields = builder.getUnknownFields();
-}
-
-private BytesBytesPair(boolean noInit) {
-this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance();
-}
-
-private static final BytesBytesPair defaultInstance;
-
-public static BytesBytesPair getDefaultInstance() {
-return defaultInstance;
-}
-
-public BytesBytesPair getDefaultInstanceForType() {
-return defaultInstance;
-}
-
-private final com.google.protobuf.UnknownFieldSet unknownFields;
-
-@java.lang.Override
-public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
-return this.unknownFields;
-}
-
-private BytesBytesPair(com.google.protobuf.CodedInputStream input, 
com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws 
com.google.protobuf.InvalidProtocolBufferException {
-initFields();
-int mutable_bitField0_ = 0;
-com.google.protobuf.UnknownFieldSet.Builder unknownFields = 
com.google.protobuf.UnknownFieldSet.newBuilder();
-try {
-boolean done = false;
-while (!done) {
-int tag = input.readTag();
-switch (tag) {
-case 0:
-done = true;
-break;
-default: {
-if (!parseUnknownField(input, unknownFields, 
extensionRegistry, tag)) {
-done = true;
-}
-break;
-}
-case 10: {
-bitField0_ |= 0x0001;
-first_ = input.readBytes();
-break;
-}
-case 18: {
-bitField0_ |= 0x0002;
-second_ = input.readBytes();
-break;
-}
-}
-}
-

[4/4] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

KYLIN-2255 drop v1 coprocessor impl


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/021a4274
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/021a4274
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/021a4274

Branch: refs/heads/KYLIN-2255
Commit: 021a427424779060181db25bce196f412751f4fe
Parents: 350fad8
Author: Li Yang 
Authored: Thu Dec 8 10:41:18 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 10:41:18 2016 +0800

--
 .../apache/kylin/query/ITCombinationTest.java   |   15 -
 .../org/apache/kylin/query/KylinTestBase.java   |2 -
 .../kylin/rest/controller/CubeController.java   |   14 -
 .../hbase/cube/v1/CubeSegmentTupleIterator.java |  344 -
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  796 ---
 .../hbase/cube/v1/CubeTupleConverter.java   |  271 
 .../hbase/cube/v1/RegionScannerAdapter.java |   97 --
 .../hbase/cube/v1/ResultScannerAdapter.java |  100 --
 .../cube/v1/SerializedHBaseTupleIterator.java   |  156 ---
 .../observer/AggregateRegionObserver.java   |  112 --
 .../observer/AggregationScanner.java|  188 ---
 .../observer/ObserverAggregationCache.java  |  166 ---
 .../observer/ObserverAggregators.java   |  265 
 .../coprocessor/observer/ObserverEnabler.java   |  191 ---
 .../v1/coprocessor/observer/ObserverTuple.java  |   71 -
 .../hbase/cube/v1/filter/FuzzyRowFilterV2.java  |  574 
 .../hbase/cube/v1/filter/UnsafeAccess.java  |  433 --
 .../v1/filter/generated/FilterProtosExt.java| 1298 --
 .../cube/v1/filter/protobuf/FilterExt.proto |   39 -
 .../cube/MeasureTypeOnlyAggrInBaseTest.java |   21 -
 .../observer/AggregateRegionObserverTest.java   |  339 -
 .../observer/RowAggregatorsTest.java|   62 -
 .../cube/v1/filter/TestFuzzyRowFilterV2.java|  249 
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  346 -
 24 files changed, 6149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/021a4274/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 496ac4e..84573b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.query.routing.Candidate;
 import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
 import org.apache.kylin.storage.hbase.HBaseStorage;
-import 
org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.runner.RunWith;
@@ -67,7 +66,6 @@ public class ITCombinationTest extends ITKylinQueryTest {
 public static Collection configs() {
 return Arrays.asList(new Object[][] { //
 { "inner", "on", "v2", true }, //
-{ "left", "on", "v1", true }, //
 { "left", "on", "v2", true }, //
 //{ "inner", "on", "v2", false }, // exclude view to simply 
model/cube selection
 //{ "left", "on", "v1", false }, // exclude view to simply 
model/cube selection
@@ -84,23 +82,10 @@ public class ITCombinationTest extends ITKylinQueryTest {
 ITKylinQueryTest.joinType = joinType;
 ITKylinQueryTest.setupAll();
 
-if (coprocessorToggle.equals("on")) {
-ObserverEnabler.forceCoprocessorOn();
-} else if (coprocessorToggle.equals("off")) {
-ObserverEnabler.forceCoprocessorOff();
-} else if (coprocessorToggle.equals("unset")) {
-// unset
-}
-
 RemoveBlackoutRealizationsRule.blackList.clear();
 if (excludeViewCubes) {
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
 
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
 }
-
-if ("v1".equalsIgnoreCase(queryEngine))
-HBaseStorage.overwriteStorageQuery = 
HBaseStorage.v1CubeStorageQuery;
-else
-HBaseStorage.overwriteStorageQuery = null;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/021a4274/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java 

[3/4] kylin git commit: KYLIN-2255 drop v1 coprocessor impl

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/021a4274/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
deleted file mode 100644
index a900ea1..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.gridtable.StorageSideBehavior;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-
-/**
- * @author yangli9
- * 
- */
-public class AggregationScanner implements RegionScanner {
-
-private RegionScanner outerScanner;
-private StorageSideBehavior behavior;
-
-public AggregationScanner(CoprocessorRowType type, CoprocessorFilter 
filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner 
innerScanner, StorageSideBehavior behavior) throws IOException {
-
-AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
-
-this.behavior = behavior;
-
-ObserverAggregationCache aggCache;
-Stats stats = new Stats();
-
-aggCache = buildAggrCache(innerScanner, type, groupBy, aggrs, filter, 
stats);
-stats.countOutputRow(aggCache.getSize());
-this.outerScanner = aggCache.getScanner(innerScanner);
-
-AggregateRegionObserver.LOG.info("Kylin Coprocessor aggregation done: 
" + stats);
-}
-
-@SuppressWarnings("rawtypes")
-ObserverAggregationCache buildAggrCache(final RegionScanner innerScanner, 
CoprocessorRowType type, CoprocessorProjector projector, ObserverAggregators 
aggregators, CoprocessorFilter filter, Stats stats) throws IOException {
-
-ObserverAggregationCache aggCache = new 
ObserverAggregationCache(aggregators);
-
-ObserverTuple tuple = new ObserverTuple(type);
-boolean hasMore = true;
-List results = new ArrayList();
-byte meaninglessByte = 0;
-
-while (hasMore) {
-results.clear();
-hasMore = innerScanner.nextRaw(results);
-if (results.isEmpty())
-continue;
-
-if (stats != null)
-stats.countInputRow(results);
-
-Cell cell = results.get(0);
-tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), 
cell.getRowLength());
-
-if (behavior == StorageSideBehavior.SCAN) {
-//touch every byte of the cell so that the cost of scanning 
will be trully reflected
-int endIndex = cell.getRowOffset() + cell.getRowLength();
-for (int i = cell.getRowOffset(); i < endIndex; ++i) {
-meaninglessByte += cell.getRowArray()[i];
-}
-} else {
-if (behavior.filterToggledOn()) {
-if (filter != null && filter.evaluate(tuple) == false)
-continue;
-
-if (behavior.aggrToggledOn()) {
-AggrKey aggKey = projector.getAggrKey(results);
-MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
-aggregators.aggregate(bufs, results);
-
-if (behavior.ordinal() >= 

[2/2] kylin git commit: minor, fix date format in sample data

[lidong]

minor, fix date format in sample data


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21bcd2f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21bcd2f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21bcd2f6

Branch: refs/heads/master
Commit: 21bcd2f6b7d56e639aba18f3f376875922c9cd68
Parents: 350fad8
Author: lidongsjtu 
Authored: Thu Dec 8 10:40:45 2016 +0800
Committer: lidongsjtu 
Committed: Thu Dec 8 10:40:55 2016 +0800

--
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 1 file changed, 1 insertions(+), 1 deletions(-)
--

[1/2] kylin git commit: minor, fix date format in sample data

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master 350fad8f2 -> 21bcd2f6b


http://git-wip-us.apache.org/repos/asf/kylin/blob/21bcd2f6/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index e13b7ad..215 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,14/12/2012,Others,88750,0,11,36.2828,4,1349,10002313,ANALYST,Beijing
-1,28/08/2012,Others,175750,0,13,23.8563,20,1927,10004376,ANALYST,Beijing
-2,16/02/2012,ABIN,148324,15,13,88.3418,18,1005,10006710,ADMIN,Shanghai
-3,19/10/2013,FP-non GTC,37831,0,13,47.3015,3,1209,10003717,ANALYST,Beijing
-4,22/10/2012,Others,140746,100,11,83.454,16,1154,10006076,ADMIN,Shanghai
-5,25/01/2013,FP-GTC,16509,0,-99,22.9896,11,1372,10007436,ADMIN,Shanghai
-6,04/04/2013,Others,963,0,13,88.5907,11,1648,10009869,MODELER,Hongkong
-7,11/04/2012,Others,15687,0,15,88.194,19,1866,1400,ADMIN,Shanghai
-8,19/01/2013,ABIN,60606,3,13,77.9727,13,1936,1687,MODELER,Hongkong
-9,30/04/2012,FP-non 
GTC,106246,0,14,52.295,17,1047,10009223,MODELER,Hongkong
-10,03/02/2013,Auction,45333,0,16,56.3584,10,1470,10006759,MODELER,Hongkong
-11,02/09/2012,FP-GTC,25147,0,-99,15.3553,8,1203,1062,ANALYST,Beijing
-12,30/07/2013,Others,24760,0,16,25.077,5,1382,10003539,ADMIN,Shanghai
-13,17/04/2012,Auction,31519,0,16,72.7384,11,1069,10001245,ADMIN,Shanghai
-14,30/04/2013,FP-GTC,51582,0,15,75.82,6,1085,10008148,ANALYST,Beijing
-15,23/03/2013,FP-GTC,16509,0,15,8.6653,15,1559,10008079,MODELER,Hongkong
-16,01/04/2013,ABIN,175750,3,16,64.2802,16,1507,10007797,MODELER,Hongkong
-17,12/07/2013,ABIN,50677,0,-99,24.5987,17,1938,10006224,ADMIN,Shanghai
-18,23/04/2012,FP-GTC,45333,0,5,72.6553,15,1863,10003824,MODELER,Hongkong
-19,19/12/2012,ABIN,3838,0,12,87.4142,4,1612,10007157,ADMIN,Shanghai
-20,15/08/2012,FP-GTC,62179,0,16,67.4238,15,1839,10003452,MODELER,Hongkong
-21,22/08/2013,Auction,1357,0,5,18.36,9,1303,1114,ANALYST,Beijing
-22,23/08/2013,ABIN,31519,0,5,67.6344,2,1877,10007417,ANALYST,Beijing
-23,12/09/2013,FP-GTC,223,0,14,71.7898,15,1848,1627,MODELER,Hongkong
-24,14/03/2012,FP-GTC,132939,0,15,26.1496,20,1815,10005292,MODELER,Hongkong
-25,13/07/2013,Others,67703,3,16,83.9661,2,1858,10008123,MODELER,Hongkong
-26,15/12/2013,FP-non GTC,100847,0,5,34.3451,16,1387,10007806,ADMIN,Shanghai
-27,30/08/2012,Auction,62179,0,11,21.0496,9,1393,10007360,ANALYST,Beijing
-28,10/01/2013,FP-GTC,48027,0,13,23.4142,20,1757,10002135,MODELER,Hongkong
-29,25/01/2013,ABIN,145970,0,5,26.7842,6,1402,10009682,ANALYST,Beijing
-30,05/07/2012,Auction,48027,0,-99,46.732,5,1972,10009522,MODELER,Hongkong
-31,07/11/2012,FP-GTC,20213,0,-99,19.727,8,1873,10009944,ANALYST,Beijing
-32,20/06/2013,ABIN,100847,0,5,76.2783,1,1409,10005395,ADMIN,Shanghai
-33,08/10/2012,ABIN,26262,0,5,48.7447,4,1029,10001836,ADMIN,Shanghai
-34,30/04/2013,ABIN,87118,0,15,63.2187,11,1529,1928,ANALYST,Beijing
-35,24/04/2013,FP-non 
GTC,20213,0,16,39.5057,19,1329,1768,ANALYST,Beijing
-36,13/05/2012,Others,164261,0,13,84.1246,15,1688,1727,ADMIN,Shanghai
-37,02/08/2013,ABIN,31519,0,5,37.1504,14,1905,1719,MODELER,Hongkong
-38,03/04/2012,Auction,164261,0,11,92.0974,13,1341,10001337,ADMIN,Shanghai
-39,02/09/2012,FP-GTC,31519,0,14,87.7829,20,1460,10008484,ANALYST,Beijing
-40,05/10/2013,ABIN,13987,0,16,74.0719,4,1381,10002975,MODELER,Hongkong
-41,13/01/2012,FP-GTC,48027,0,15,71.2637,14,1915,10005778,ADMIN,Shanghai
-42,01/03/2013,Others,13836,0,14,16.7288,14,1953,10009385,ADMIN,Shanghai
-43,12/09/2012,FP-non 
GTC,16509,0,5,12.2933,20,1277,10002286,MODELER,Hongkong
-44,07/02/2012,Auction,45333,0,5,64.977,16,1040,10005583,MODELER,Hongkong
-45,14/12/2013,FP-non 
GTC,158798,0,16,72.4413,18,1500,10007635,MODELER,Hongkong
-46,13/10/2013,Auction,31519,0,5,79.3053,17,1816,10006423,MODELER,Hongkong
-47,11/06/2012,ABIN,4943,0,5,11.6942,15,1696,10001237,ANALYST,Beijing
-48,18/10/2012,ABIN,80053,0,-99,54.0933,4,1604,10006851,MODELER,Hongkong
-49,03/11/2012,Auction,20886,0,5,9.8258,8,1764,10001220,MODELER,Hongkong
-50,15/01/2012,Auction,44079,0,14,13.0371,19,1343,10001259,MODELER,Hongkong
-51,03/07/2012,FP-GTC,159184,0,-99,92.5314,4,1724,10009008,ADMIN,Shanghai
-52,16/02/2012,FP-GTC,314,0,13,49.1825,20,1334,10004005,ADMIN,Shanghai
-53,17/06/2012,FP-non GTC,31387,3,12,0.6677,5,1603,10002595,ANALYST,Beijing
-54,15/07/2012,FP-non 
GTC,32876,0,-99,50.9634,11,1248,10006890,ANALYST,Beijing
-55,24/07/2012,FP-GTC,150265,15,16,57.5645,6,1653,10007269,ADMIN,Shanghai
-56,06/10/2012,Auction,145970,0,5,44.3091,16,1082,10001640,ADMIN,Shanghai

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2254 [deleted] 8ba8aea6e

kylin git commit: KYLIN-2254 allow 2 columns in a CompareTupleFilter

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master b1fd4679f -> 350fad8f2


KYLIN-2254 allow 2 columns in a CompareTupleFilter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/350fad8f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/350fad8f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/350fad8f

Branch: refs/heads/master
Commit: 350fad8f248988596c05b080b3c5a68b77b790a1
Parents: b1fd467
Author: Li Yang 
Authored: Wed Dec 7 17:51:26 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 09:47:27 2016 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 23 ++--
 .../kylin/metadata/model/FunctionDesc.java  |  2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |  2 +-
 .../resources/query/sql_subquery/query12.sql| 37 
 .../kylin/query/relnode/OLAPAggregateRel.java   | 13 ---
 .../kylin/query/relnode/OLAPFilterRel.java  | 30 +---
 6 files changed, 66 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..d567c89 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -37,6 +37,7 @@ public class CompareTupleFilter extends TupleFilter {
 // operand 1 is either a column or a function
 private TblColRef column;
 private FunctionTupleFilter function;
+private TblColRef secondColumn;
 
 // operand 2 is constants
 private Set conditionValues;
@@ -73,15 +74,16 @@ public class CompareTupleFilter extends TupleFilter {
 if (child instanceof ColumnTupleFilter) {
 ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
 if (this.column != null) {
-throw new IllegalStateException("Duplicate columns! old is " + 
column.getName() + " and new is " + columnFilter.getColumn().getName());
-}
-this.column = columnFilter.getColumn();
-// if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
-// children.size() > 1 means already added one conditionValue or 
dynamicVariable
-if (this.children.size() > 1 && needSwapOperator()) {
-this.operator = SWAP_OP_MAP.get(this.operator);
-TupleFilter last = this.children.remove(this.children.size() - 
1);
-this.children.add(0, last);
+this.secondColumn = columnFilter.getColumn();
+} else {
+this.column = columnFilter.getColumn();
+// if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
+// children.size() > 1 means already added one conditionValue 
or dynamicVariable
+if (this.children.size() > 1 && needSwapOperator()) {
+this.operator = SWAP_OP_MAP.get(this.operator);
+TupleFilter last = 
this.children.remove(this.children.size() - 1);
+this.children.add(0, last);
+}
 }
 } else if (child instanceof ConstantTupleFilter) {
 this.conditionValues.addAll(child.getValues());
@@ -212,7 +214,8 @@ public class CompareTupleFilter extends TupleFilter {
 
 @Override
 public boolean isEvaluable() {
-return ((function != null && function.isEvaluable()) || column != 
null) && !conditionValues.isEmpty();
+return (column != null || (function != null && 
function.isEvaluable())) //
+&& !conditionValues.isEmpty() && secondColumn == null;
 }
 
 @SuppressWarnings({ "unchecked", "rawtypes" })

http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 832cb4a..9252c42 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -140,7 +140,7 @@ public class FunctionDesc {
 } else if (isCount()) {
 return 

kylin git commit: KYLIN-2254 allow 2 columns in a CompareTupleFilter

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2254 3e2d108e1 -> 8ba8aea6e


KYLIN-2254 allow 2 columns in a CompareTupleFilter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ba8aea6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ba8aea6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ba8aea6

Branch: refs/heads/KYLIN-2254
Commit: 8ba8aea6e7e130d783edcc547bd09816cdfae233
Parents: 3e2d108
Author: Li Yang 
Authored: Thu Dec 8 09:46:38 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 8 09:46:38 2016 +0800

--
 .../metadata/filter/CompareTupleFilter.java | 23 +++-
 1 file changed, 13 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8ba8aea6/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..d567c89 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -37,6 +37,7 @@ public class CompareTupleFilter extends TupleFilter {
 // operand 1 is either a column or a function
 private TblColRef column;
 private FunctionTupleFilter function;
+private TblColRef secondColumn;
 
 // operand 2 is constants
 private Set conditionValues;
@@ -73,15 +74,16 @@ public class CompareTupleFilter extends TupleFilter {
 if (child instanceof ColumnTupleFilter) {
 ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
 if (this.column != null) {
-throw new IllegalStateException("Duplicate columns! old is " + 
column.getName() + " and new is " + columnFilter.getColumn().getName());
-}
-this.column = columnFilter.getColumn();
-// if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
-// children.size() > 1 means already added one conditionValue or 
dynamicVariable
-if (this.children.size() > 1 && needSwapOperator()) {
-this.operator = SWAP_OP_MAP.get(this.operator);
-TupleFilter last = this.children.remove(this.children.size() - 
1);
-this.children.add(0, last);
+this.secondColumn = columnFilter.getColumn();
+} else {
+this.column = columnFilter.getColumn();
+// if value is before column, we need to reverse the operator. 
e.g. "1 >= c1" => "c1 <= 1"
+// children.size() > 1 means already added one conditionValue 
or dynamicVariable
+if (this.children.size() > 1 && needSwapOperator()) {
+this.operator = SWAP_OP_MAP.get(this.operator);
+TupleFilter last = 
this.children.remove(this.children.size() - 1);
+this.children.add(0, last);
+}
 }
 } else if (child instanceof ConstantTupleFilter) {
 this.conditionValues.addAll(child.getValues());
@@ -212,7 +214,8 @@ public class CompareTupleFilter extends TupleFilter {
 
 @Override
 public boolean isEvaluable() {
-return ((function != null && function.isEvaluable()) || column != 
null) && !conditionValues.isEmpty();
+return (column != null || (function != null && 
function.isEvaluable())) //
+&& !conditionValues.isEmpty() && secondColumn == null;
 }
 
 @SuppressWarnings({ "unchecked", "rawtypes" })

kylin git commit: minor refactor

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2254 9af1e264f -> 3e2d108e1


minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3e2d108e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3e2d108e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3e2d108e

Branch: refs/heads/KYLIN-2254
Commit: 3e2d108e1efb2e5faf8f3d405aae3498a33500c3
Parents: 9af1e26
Author: Yang Li 
Authored: Thu Dec 8 07:23:49 2016 +0800
Committer: Yang Li 
Committed: Thu Dec 8 07:23:49 2016 +0800

--
 .../kylin/metadata/model/FunctionDesc.java  |  2 +-
 .../kylin/query/relnode/OLAPAggregateRel.java   | 13 
 .../kylin/query/relnode/OLAPFilterRel.java  | 35 +---
 3 files changed, 15 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3e2d108e/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 832cb4a..9252c42 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -140,7 +140,7 @@ public class FunctionDesc {
 } else if (isCount()) {
 return "COUNT__"; // ignores parameter, count(*), count(1), 
count(col) are all the same
 } else {
-return getFullExpression().replaceAll("[(), ]", "_");
+return getFullExpression().replaceAll("[(),. ]", "_");
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3e2d108e/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 24933f5..f5fa74d 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -192,17 +192,16 @@ public class OLAPAggregateRel extends Aggregate 
implements OLAPRel {
 
 for (int i = 0; i < this.aggregations.size(); i++) {
 FunctionDesc aggFunc = this.aggregations.get(i);
-TblColRef aggCol = null;
+String aggOutName;
 if (aggFunc.needRewriteField()) {
-aggCol = buildRewriteColumn(aggFunc);
+aggOutName = aggFunc.getRewriteFieldName();
 } else {
 AggregateCall aggCall = this.rewriteAggCalls.get(i);
-if (!aggCall.getArgList().isEmpty()) {
-int index = aggCall.getArgList().get(0);
-aggCol = inputColumnRowType.getColumnByIndex(index);
-}
+int index = aggCall.getArgList().get(0);
+aggOutName = aggFunc.getExpression() + "_" + 
inputColumnRowType.getColumnByIndex(index).getIdentity() + "_";
 }
-columns.add(aggCol);
+TblColRef aggOutCol = TblColRef.newInnerColumn(aggOutName, 
TblColRef.InnerDataTypeEnum.LITERAL);
+columns.add(aggOutCol);
 }
 return new ColumnRowType(columns);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3e2d108e/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 9573dfb..8023df4 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -20,6 +20,7 @@ package org.apache.kylin.query.relnode;
 
 import java.util.GregorianCalendar;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -61,14 +62,13 @@ import org.apache.kylin.metadata.filter.DynamicTupleFilter;
 import org.apache.kylin.metadata.filter.ExtractTupleFilter;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
 import org.apache.kylin.metadata.filter.function.Functions;
 import 

kylin git commit: KYLIN-UNKNOWN, sub_query/query12.sql

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2254 [created] 9af1e264f


KYLIN-UNKNOWN, sub_query/query12.sql


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9af1e264
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9af1e264
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9af1e264

Branch: refs/heads/KYLIN-2254
Commit: 9af1e264ff29441c409737a78f4c2c33082f04a6
Parents: 365caae
Author: Li Yang 
Authored: Wed Dec 7 17:51:26 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 18:29:51 2016 +0800

--
 .../apache/kylin/metadata/model/TblColRef.java  |  2 +-
 .../resources/query/sql_subquery/query12.sql| 37 
 .../kylin/query/relnode/OLAPFilterRel.java  |  9 +++--
 3 files changed, 45 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9af1e264/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 5d72c3f..fd0224f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -150,7 +150,7 @@ public class TblColRef implements Serializable {
 return column.getType();
 }
 
-public void markInnerColumn(InnerDataTypeEnum dataType) {
+private void markInnerColumn(InnerDataTypeEnum dataType) {
 this.column.setDatatype(dataType.getDataType());
 this.column.getTable().setName(INNER_TABLE_NAME);
 this.column.getTable().setDatabase("DEFAULT");

http://git-wip-us.apache.org/repos/asf/kylin/blob/9af1e264/kylin-it/src/test/resources/query/sql_subquery/query12.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query12.sql 
b/kylin-it/src/test/resources/query/sql_subquery/query12.sql
new file mode 100644
index 000..6ed45fa
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query12.sql
@@ -0,0 +1,37 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+  f.lstg_format_name
+  ,sum(price) as sum_price
+FROM
+  test_kylin_fact f
+  inner join
+  ( 
+select
+  lstg_format_name,
+  min(slr_segment_cd) as min_seg
+from
+  test_kylin_fact
+group by
+  lstg_format_name
+  ) t on f.lstg_format_name = t.lstg_format_name
+where
+  f.slr_segment_cd = min_seg
+group by
+  f.lstg_format_name

http://git-wip-us.apache.org/repos/asf/kylin/blob/9af1e264/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 411142d..9573dfb 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -151,7 +151,11 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 if (filter == null) {
 filter = cast(childFilter, call.type);
 } else {
-filter.addChild(childFilter);
+try {
+filter.addChild(childFilter);
+} catch (Exception ex) {
+return new 
UnsupportedTupleFilter(FilterOperatorEnum.UNSUPPORTED);
+}
 }
 }
 
@@ -227,7 +231,8 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 @Override
 public TupleFilter visitInputRef(RexInputRef inputRef) {
 TblColRef column = 
inputRowType.getColumnByIndex(inputRef.getIndex());
- 

kylin git commit: minor, add API to get AggregationGroup combination cuboid num

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 365caaebd -> b1fd4679f


minor, add API to get AggregationGroup combination cuboid num


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1fd4679
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1fd4679
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1fd4679

Branch: refs/heads/master
Commit: b1fd4679fa80e8e971d553a42baf20198afb9eac
Parents: 365caae
Author: Billy Liu 
Authored: Wed Dec 7 18:29:07 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 18:29:07 2016 +0800

--
 .../kylin/rest/controller/CubeController.java   | 33 
 1 file changed, 33 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b1fd4679/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f537231..ab32551 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -655,6 +656,22 @@ public class CubeController extends BasicController {
 return response;
 }
 
+/**
+ * Calculate Cuboid Combination based on the AggreationGroup definition.
+ *
+ * @param aggregationGroupStr
+ * @return number of cuboid, -1 if failed
+ */
+@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.POST)
+@ResponseBody
+public long calculateCuboidCombination(@RequestBody String 
aggregationGroupStr) {
+AggregationGroup aggregationGroup = 
deserializeAggregationGroup(aggregationGroupStr);
+if (aggregationGroup != null) {
+return aggregationGroup.calculateCuboidCombination();
+} else
+return -1;
+}
+
 private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
 CubeDesc desc = null;
 try {
@@ -673,6 +690,22 @@ public class CubeController extends BasicController {
 return desc;
 }
 
+private AggregationGroup deserializeAggregationGroup(String 
aggregationGroupStr) {
+AggregationGroup aggreationGroup = null;
+try {
+logger.debug("Parsing AggregationGroup " + aggregationGroupStr);
+aggreationGroup = JsonUtil.readValue(aggregationGroupStr, 
AggregationGroup.class);
+} catch (JsonParseException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (JsonMappingException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (IOException e) {
+logger.error("Failed to deal with the request.", e);
+throw new InternalErrorException("Failed to deal with the 
request:" + e.getMessage(), e);
+}
+return aggreationGroup;
+}
+
 private void updateRequest(CubeRequest request, boolean success, String 
message) {
 request.setCubeDescData("");
 request.setSuccessful(success);

[2/6] kylin git commit: KYLIN-1875 fix ParameterDesc equals() & hashcode()

[liyang]

KYLIN-1875 fix ParameterDesc equals() & hashcode()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/227bbf41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/227bbf41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/227bbf41

Branch: refs/heads/master
Commit: 227bbf4152074c0621bf2b1047102d416d4dd3eb
Parents: 967ef18
Author: Yang Li 
Authored: Thu Dec 1 06:58:08 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:36 2016 +0800

--
 .../kylin/metadata/model/FunctionDesc.java  |  1 +
 .../kylin/metadata/model/ParameterDesc.java | 36 +++-
 2 files changed, 28 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/227bbf41/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index dfa6f3b..b9e5543 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -248,6 +248,7 @@ public class FunctionDesc {
 final int prime = 31;
 int result = 1;
 result = prime * result + ((expression == null) ? 0 : 
expression.hashCode());
+result = prime * result + ((returnType == null) ? 0 : 
returnType.hashCode());
 result = prime * result + ((isCount() || parameter == null) ? 0 : 
parameter.hashCode());
 return result;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/227bbf41/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 329799f..4a95fea 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -22,8 +22,10 @@ import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableList;
 
 /**
  */
@@ -36,9 +38,10 @@ public class ParameterDesc {
 private String value;
 
 @JsonProperty("next_parameter")
+@JsonInclude(JsonInclude.Include.NON_NULL)
 private ParameterDesc nextParameter;
 
-private List colRefs;
+private List colRefs = ImmutableList.of();
 
 public String getType() {
 return type;
@@ -89,21 +92,36 @@ public class ParameterDesc {
 
 ParameterDesc that = (ParameterDesc) o;
 
-if (nextParameter != null ? !nextParameter.equals(that.nextParameter) 
: that.nextParameter != null)
-return false;
 if (type != null ? !type.equals(that.type) : that.type != null)
 return false;
-if (value != null ? !value.equals(that.value) : that.value != null)
-return false;
-
-return true;
+
+ParameterDesc p = this, q = that;
+int refi = 0, refj = 0;
+for (; p != null && q != null; p = p.nextParameter, q = 
q.nextParameter) {
+if (p.isColumnType()) {
+if (q.isColumnType() == false)
+return false;
+if (refi >= this.colRefs.size() || refj >= that.colRefs.size())
+return false;
+if (this.colRefs.get(refi).equals(that.colRefs.get(refj)) == 
false)
+return false;
+refi++;
+refj++;
+} else {
+if (q.isColumnType() == true)
+return false;
+if (p.value.equals(q.value) == false)
+return false;
+}
+}
+
+return p == null && q == null;
 }
 
 @Override
 public int hashCode() {
 int result = type != null ? type.hashCode() : 0;
-result = 31 * result + (value != null ? value.hashCode() : 0);
-result = 31 * result + (nextParameter != null ? 
nextParameter.hashCode() : 0);
+result = 31 * result + (colRefs != null ? colRefs.hashCode() : 0);
 return result;
 }
 

[3/6] kylin git commit: KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc

[liyang]

KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64d9b8b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64d9b8b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64d9b8b8

Branch: refs/heads/master
Commit: 64d9b8b89f84cc31f9931e5a6cdf89bdfa13925a
Parents: 227bbf4
Author: Li Yang 
Authored: Thu Dec 1 14:10:44 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:37 2016 +0800

--
 .../apache/kylin/cube/RawQueryLastHacker.java   |  8 +---
 .../metadata/measure/MeasureCodecTest.java  |  3 +-
 .../kylin/measure/raw/RawMeasureType.java   | 12 ++---
 .../kylin/metadata/model/FunctionDesc.java  | 26 ---
 .../kylin/metadata/model/ParameterDesc.java | 47 +++-
 .../apache/kylin/storage/StorageMockUtils.java  | 32 +++--
 .../kylin/query/relnode/OLAPAggregateRel.java   | 12 ++---
 7 files changed, 59 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index b0a4823..682e48c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -57,12 +57,8 @@ public class RawQueryLastHacker {
 sqlDigest.groupbyColumns.add(col);
 } else {
 // For measure columns, take them as metric columns with 
aggregation function SUM().
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc sumFunc = new FunctionDesc();
-sumFunc.setExpression("SUM");
-sumFunc.setParameter(colParameter);
+ParameterDesc parameter = ParameterDesc.newInstance(col);
+FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", 
parameter, null);
 
 boolean measureHasSum = false;
 for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 6bb71e9..18680ec 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -83,8 +83,7 @@ public class MeasureCodecTest extends 
LocalFileMetadataTestCase {
 
 private MeasureDesc measure(String returnType) {
 MeasureDesc desc = new MeasureDesc();
-FunctionDesc func = new FunctionDesc();
-func.setReturnType(returnType);
+FunctionDesc func = FunctionDesc.newInstance(null, null, returnType);
 desc.setFunction(func);
 return desc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 3a49d31..a5bb06b 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -197,18 +197,12 @@ public class RawMeasureType extends 
MeasureType {
 if (sqlDigest.isRawQuery) {
 for (MeasureDesc measureDesc : measureDescs) {
 TblColRef col = this.getRawColumn(measureDesc.getFunction());
-ParameterDesc colParameter = new ParameterDesc();
-colParameter.setType("column");
-colParameter.setValue(col.getName());
-FunctionDesc rawFunc = new FunctionDesc();
-rawFunc.setExpression("RAW");
-rawFunc.setParameter(colParameter);
+ParameterDesc colParameter = ParameterDesc.newInstance(col);
+FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", 
colParameter, null);
 
 if 

[5/6] kylin git commit: KYLIN-1875 fix PK-FK derive from each other

[liyang]

KYLIN-1875 fix PK-FK derive from each other


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92e4d464
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92e4d464
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92e4d464

Branch: refs/heads/master
Commit: 92e4d464069745f0c58cc0559e054efa470960c1
Parents: 79e4801
Author: Yang Li 
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:39 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  8 +--
 .../org/apache/kylin/cube/model/CubeDesc.java   | 52 +++-
 .../storage/gtrecord/CubeTupleConverter.java|  2 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  2 +-
 .../translate/DerivedFilterTranslator.java  |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../query/enumerator/LookupTableEnumerator.java |  2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  2 +-
 .../hbase/cube/v1/CubeTupleConverter.java   |  2 +-
 9 files changed, 41 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index bda1423..fe2030a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -44,7 +44,6 @@ import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -54,6 +53,7 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -629,10 +629,10 @@ public class CubeManager implements IRealizationProvider {
 cubeMap.removeLocal(cubeName);
 }
 
-public LookupStringTable getLookupTable(CubeSegment cubeSegment, 
DimensionDesc dim) {
+public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc 
join) {
 
-String tableName = dim.getTableRef().getTableIdentity();
-String[] pkCols = dim.getJoin().getPrimaryKey();
+String tableName = join.getPKSide().getTableIdentity();
+String[] pkCols = join.getPrimaryKey();
 String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
 if (snapshotResPath == null)
 throw new IllegalStateException("No snaphot for table '" + 
tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() 
+ "/" + cubeSegment);

http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f95cceb..f8c316c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.IEngineAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectInstance;
@@ -101,20 +102,20 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public static class DeriveInfo {
 public DeriveType type;
-public DimensionDesc dimension;
+public JoinDesc join;
 public TblColRef[] columns;
 public boolean isOneToOne; // only used when ref from derived to host
 
-DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] 
columns, boolean isOneToOne) {
+DeriveInfo(DeriveType type, JoinDesc join, 

[6/6] kylin git commit: KYLIN-1875 minor, test fixes

[liyang]

KYLIN-1875 minor, test fixes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/365caaeb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/365caaeb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/365caaeb

Branch: refs/heads/master
Commit: 365caaebd841917f161dab37b0af7c055626194c
Parents: 92e4d46
Author: Yang Li 
Authored: Fri Dec 2 06:32:25 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:46 2016 +0800

--
 .../metadata/filter/ColumnTupleFilter.java  |  1 +
 .../kylin/metadata/model/DataModelDesc.java |  2 +-
 .../apache/kylin/metadata/model/TblColRef.java  |  2 +-
 .../template/model_desc/kylin_sales_model.json  |  4 +++
 .../apache/kylin/query/ITKylinQueryTest.java|  2 +-
 .../resources/query/sql_subquery/query02.sql| 31 
 .../query/sql_subquery/query02.sql.disable  | 25 
 7 files changed, 39 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index c8a8f07..ecb8e61 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -121,6 +121,7 @@ public class ColumnTupleFilter extends TupleFilter {
 }
 }
 
+@SuppressWarnings("deprecation")
 @Override
 public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 898ff74..3f868a2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -112,7 +112,7 @@ public class DataModelDesc extends RootPersistentEntity {
 return name;
 }
 
-// for test only
+// for test mainly
 @Deprecated
 public void setName(String name) {
 this.name = name;

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index bf8d36b..5d72c3f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -84,7 +84,7 @@ public class TblColRef implements Serializable {
 col.table = tableRef;
 }
 
-// for test only
+// for test mainly
 public static TblColRef mockup(TableDesc table, int oneBasedColumnIndex, 
String name, String datatype) {
 ColumnDesc desc = new ColumnDesc();
 String id = "" + oneBasedColumnIndex;

http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 2d3cfb6..cce360f 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -18,6 +18,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_ACCOUNT",
 "alias" : "BUYER_ACCOUNT",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "ACCOUNT_ID" ],
@@ -26,6 +27,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_ACCOUNT",
 "alias" : "SELLER_ACCOUNT",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "ACCOUNT_ID" ],
@@ -34,6 +36,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_COUNTRY",
 "alias" : "BUYER_COUNTRY",
+"kind" : "LOOKUP",
 "join" : {
   "type" : "inner",
   "primary_key" : [ "COUNTRY" ],
@@ -42,6 +45,7 @@
   }, {
 "table" : "DEFAULT.KYLIN_COUNTRY",
 "alias" : "SELLER_COUNTRY",
+"kind" : "LOOKUP",
 

[4/6] kylin git commit: KYLIN-1875 Support measure on non-root-fact-table

[liyang]

KYLIN-1875 Support measure on non-root-fact-table


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79e48011
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79e48011
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79e48011

Branch: refs/heads/master
Commit: 79e48011bd3f4696a916f0bfee244ec28355f3b3
Parents: 64d9b8b
Author: Li Yang 
Authored: Thu Dec 1 18:59:08 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:38 2016 +0800

--
 .../kylin/metadata/project/ProjectL2Cache.java  | 26 +---
 .../template/cube_desc/kylin_sales_cube.json| 18 +++---
 .../kylin/query/relnode/OLAPTableScan.java  |  4 +++
 3 files changed, 30 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 82c0de3..14d7843 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
@@ -126,17 +127,17 @@ class ProjectL2Cache {
 return Collections.unmodifiableSet(tableCache.realizations);
 }
 
-public List listEffectiveRewriteMeasures(String project, 
String factTable, boolean onlyRewriteMeasure) {
-Set realizations = getRealizationsByTable(project, 
factTable);
+public List listEffectiveRewriteMeasures(String project, 
String table, boolean onlyRewriteMeasure) {
+Set realizations = getRealizationsByTable(project, 
table);
 List result = Lists.newArrayList();
 for (IRealization r : realizations) {
-if (r.getModel().isFactTable(factTable) && r.isReady()) {
-for (MeasureDesc m : r.getMeasures()) {
-FunctionDesc func = m.getFunction();
-if (onlyRewriteMeasure) {
-if (func.needRewrite())
-result.add(m);
-} else {
+if (!r.isReady())
+continue;
+
+for (MeasureDesc m : r.getMeasures()) {
+FunctionDesc func = m.getFunction();
+if (belongToTable(func, table, r.getModel())) {
+if (!onlyRewriteMeasure || func.needRewrite()) {
 result.add(m);
 }
 }
@@ -145,6 +146,13 @@ class ProjectL2Cache {
 return result;
 }
 
+private boolean belongToTable(FunctionDesc func, String table, 
DataModelDesc model) {
+// measure belong to the first column parameter's table
+List cols = func.getParameter().getColRefs();
+String belongTo = cols.isEmpty() ? 
model.getRootFactTable().getTableIdentity() : cols.get(0).getTable();
+return belongTo.equals(table);
+}
+
 // 

 // build the cache
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 48eef46..2a27305 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -103,26 +103,26 @@
   "returntype" : "decimal(19,4)"
 }
   }, {
-"name" : "GMV_MIN",
+"name" : "BUYER_LEVEL_SUM",
 "function" : {
-  "expression" : "MIN",
+  "expression" : "SUM",
   "parameter" : {
 "type" : "column",
-"value" : "PRICE",
+"value" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
 "next_parameter" : null
   },
-  "returntype" : "decimal(19,4)"
+  "returntype" : "bigint"
 }
   }, {
-"name" : "GMV_MAX",
+"name" : "SELLER_LEVEL_SUM",
 "function" : {
-  "expression" : "MAX",
+  "expression" : 

[1/6] kylin git commit: KYLIN-1875 Normalize table and column names in JSON

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 081ed0d50 -> 365caaebd


KYLIN-1875 Normalize table and column names in JSON


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/967ef180
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/967ef180
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/967ef180

Branch: refs/heads/master
Commit: 967ef18062048e199cd0dd351ba82618a10b08e5
Parents: 081ed0d
Author: Li Yang 
Authored: Tue Nov 29 20:32:04 2016 +0800
Committer: Li Yang 
Committed: Wed Dec 7 17:53:35 2016 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   | 22 +---
 .../org/apache/kylin/cube/model/CubeDesc.java   |  1 +
 .../apache/kylin/cube/model/DimensionDesc.java  | 19 +-
 .../model/validation/rule/FunctionRule.java | 37 
 .../kylin/metadata/model/DataModelDesc.java | 13 ++-
 .../kylin/metadata/model/FunctionDesc.java  |  5 +--
 .../metadata/model/ModelDimensionDesc.java  | 19 +++---
 .../kylin/metadata/model/PartitionDesc.java |  4 +--
 8 files changed, 64 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 677b713..011035b 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -81,8 +81,7 @@ public class FactTableGenerator {
 // table(appear as fk in fact table)
 TreeMap lookupTableKeys = new 
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
-// possible values of lookupTableKeys, extracted from existing lookup
-// tables.
+// possible values of lookupTableKeys, extracted from existing lookup 
tables.
 // The key is in the format of tablename/columnname
 TreeMap feasibleValues = new 
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
@@ -244,7 +243,7 @@ public class FactTableGenerator {
 JoinDesc join = dim.getJoin();
 if (join != null) {
 String lookupTable = dim.getTableRef().getTableIdentity();
-for (String column : join.getPrimaryKey()) {
+for (String column : dropAlias(join.getPrimaryKey())) {
 if (!lookupTableKeys.containsKey(lookupTable)) {
 lookupTableKeys.put(lookupTable, new 
LinkedList());
 }
@@ -297,8 +296,8 @@ public class FactTableGenerator {
 for (DimensionDesc dim : dimensions) {
 JoinDesc jDesc = dim.getJoin();
 if (jDesc != null) {
-String[] fks = jDesc.getForeignKey();
-String[] pks = jDesc.getPrimaryKey();
+String[] fks = dropAlias(jDesc.getForeignKey());
+String[] pks = dropAlias(jDesc.getPrimaryKey());
 int num = fks.length;
 for (int i = 0; i < num; ++i) {
 String value = dim.getTableRef().getTableIdentity() + "/" 
+ pks[i];
@@ -343,6 +342,19 @@ public class FactTableGenerator {
 return createTable(this.rowCount, factTableCol2LookupCol, 
lookupCol2factTableCol, usedCols);
 }
 
+private String[] dropAlias(String[] aliasDotCol) {
+String[] result = new String[aliasDotCol.length];
+for (int i = 0; i < aliasDotCol.length; i++) {
+String str = aliasDotCol[i];
+int cut = str.lastIndexOf('.');
+if (cut >= 0) {
+str = str.substring(cut + 1);
+}
+result[i] = str;
+}
+return result;
+}
+
 private String normToTwoDigits(int v) {
 if (v < 10)
 return "0" + v;

http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 853571c..f95cceb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -543,6 +543,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 initMeasureColumns();
 
 rowkey.init(this);
+
 validateAggregationGroups(); // check if aggregation group is 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/e7987218/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/e7987218/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[lidong]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6144e8ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6144e8ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6144e8ec

Branch: refs/heads/yang22-cdh5.7
Commit: 6144e8ec064e199a347e096289be4c7514d00b7d
Parents: e798721
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Wed Dec 7 16:12:02 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6144e8ec/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/6144e8ec/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6144e8ec/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7987218
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7987218
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7987218

Branch: refs/heads/yang22-cdh5.7
Commit: e7987218269c3bfa531dbebe366d501b975d4847
Parents: 43805bc
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 16:10:38 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e7987218/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-   

[1/3] kylin git commit: minor, update calculateCuboid API to POST [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 838621385 -> e79872182 (forced update)


minor, update calculateCuboid API to POST


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/43805bcd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/43805bcd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/43805bcd

Branch: refs/heads/yang22-hbase1.x
Commit: 43805bcd2c3bb49d5b1c433f6205927d36a7eaad
Parents: 8a42973
Author: Billy Liu 
Authored: Wed Dec 7 15:49:17 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 15:49:17 2016 +0800

--
 .../main/java/org/apache/kylin/rest/controller/CubeController.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/43805bcd/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8dce112..4c07f7c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -652,7 +652,7 @@ public class CubeController extends BasicController {
  * @param aggregationGroupStr
  * @return number of cuboid, -1 if failed
  */
-@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.GET)
+@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.POST)
 @ResponseBody
 public long calculateCuboidCombination(@RequestBody String 
aggregationGroupStr) {
 AggregationGroup aggregationGroup = 
deserializeAggregationGroup(aggregationGroupStr);