kylin git commit: KYLIN 1875 change limited table name

2017-04-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/2.0.x accb417b9 -> f885c8d73


KYLIN 1875 change limited table name

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f885c8d7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f885c8d7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f885c8d7

Branch: refs/heads/2.0.x
Commit: f885c8d73798b7c90b934eaeaa3cb51f02259370
Parents: accb417
Author: chenzhx <346839...@qq.com>
Authored: Mon Apr 10 17:02:32 2017 +0800
Committer: Li Yang 
Committed: Wed Apr 12 10:40:24 2017 +0800

--
 .../app/partials/modelDesigner/data_model.html  | 24 +++-
 1 file changed, 13 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f885c8d7/webapp/app/partials/modelDesigner/data_model.html
--
diff --git a/webapp/app/partials/modelDesigner/data_model.html 
b/webapp/app/partials/modelDesigner/data_model.html
index ca93c90..cff0b06 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -130,9 +130,9 @@
 
   
 
-  
+  
   
   From Table 
 
@@ -140,28 +140,30 @@
   
 
 
-  
+  
   
-  
 
   
   
 
 
-  
-  
+  
   Select Lookup 
Table  
   
-  
+  
+  
+AS
+  
+  
 
  Table alias is invalid(A Combination 
of numbers, uppercase letters or underscores).
-  
-  AS
   
 
   
@@ -170,7 +172,7 @@
 
   
 
-  Is Limited
+  Skip snapshot for this 
lookup table.
 
 
   
@@ -239,8 +241,8 @@
 
 
 
+Cancel
 OK
-Cancel

kylin git commit: KYLIN 1875 change limited table name

2017-04-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master e9ec116c2 -> 1caf19ab5


KYLIN 1875 change limited table name

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1caf19ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1caf19ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1caf19ab

Branch: refs/heads/master
Commit: 1caf19ab5f7f309899f93b9f6c426db11a6ff19a
Parents: e9ec116
Author: chenzhx <346839...@qq.com>
Authored: Mon Apr 10 17:02:32 2017 +0800
Committer: Li Yang 
Committed: Wed Apr 12 10:39:55 2017 +0800

--
 .../app/partials/modelDesigner/data_model.html  | 24 +++-
 1 file changed, 13 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1caf19ab/webapp/app/partials/modelDesigner/data_model.html
--
diff --git a/webapp/app/partials/modelDesigner/data_model.html 
b/webapp/app/partials/modelDesigner/data_model.html
index ca93c90..cff0b06 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -130,9 +130,9 @@
 
   
 
-  
+  
   
   From Table 
 
@@ -140,28 +140,30 @@
   
 
 
-  
+  
   
-  
 
   
   
 
 
-  
-  
+  
   Select Lookup 
Table  
   
-  
+  
+  
+AS
+  
+  
 
  Table alias is invalid(A Combination 
of numbers, uppercase letters or underscores).
-  
-  AS
   
 
   
@@ -170,7 +172,7 @@
 
   
 
-  Is Limited
+  Skip snapshot for this 
lookup table.
 
 
   
@@ -239,8 +241,8 @@
 
 
 
+Cancel
 OK
-Cancel

kylin git commit: minor, but important, backport select * fix from 2.0 branch

2017-04-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang22 a943ddaa2 -> c2d733ef3


minor, but important, backport select * fix from 2.0 branch


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2d733ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2d733ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2d733ef

Branch: refs/heads/yang22
Commit: c2d733ef3ba48e41622b743bd0fd4325fddf1292
Parents: a943dda
Author: Li Yang 
Authored: Tue Apr 11 19:02:18 2017 +0800
Committer: Li Yang 
Committed: Tue Apr 11 19:02:18 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 34 ++--
 1 file changed, 17 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2d733ef/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 927bc82..0b4bee6 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -596,14 +596,14 @@ public class SqlToRelConverter {
 
 /* OVERRIDE POINT */
 private RelRoot hackSelectStar(SqlNode query, RelRoot root) {
-/*
- * Rel tree is like:
- *
- *   LogicalSort (optional)
- *|- LogicalProject
- *|- LogicalFilter (optional)
- *|- OLAPTableScan or LogicalJoin
- */
+///*
+// * Rel tree is like:
+// *
+// *   LogicalSort (optional)
+// *|- LogicalProject
+// *|- LogicalFilter (optional)
+// *|- OLAPTableScan or LogicalJoin
+// */
 LogicalProject rootPrj = null;
 LogicalSort rootSort = null;
 if (root.rel instanceof LogicalProject) {
@@ -614,16 +614,16 @@ public class SqlToRelConverter {
 } else {
 return root;
 }
-
+//
 RelNode input = rootPrj.getInput();
-if (!(//
-isAmong(input, "OLAPTableScan", "LogicalJoin")//
-|| (isAmong(input, "LogicalFilter") && 
isAmong(input.getInput(0), "OLAPTableScan", "LogicalJoin"))//
-))
-return root;
-
-if (rootPrj.getRowType().getFieldCount() < 
input.getRowType().getFieldCount())
-return root;
+//if (!(//
+//isAmong(input, "OLAPTableScan", "LogicalJoin")//
+//|| (isAmong(input, "LogicalFilter") && 
isAmong(input.getInput(0), "OLAPTableScan", "LogicalJoin"))//
+// ))
+//return root;
+//
+//if (rootPrj.getRowType().getFieldCount() < 
input.getRowType().getFieldCount())
+//return root;
 
 RelDataType inType = rootPrj.getRowType();
 List inFields = inType.getFieldNames();

[4/9] kylin git commit: minor, fix check style error

2017-04-11 Thread liyang 
minor, fix check style error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/88e66f3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/88e66f3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/88e66f3e

Branch: refs/heads/master
Commit: 88e66f3ea642816c91c964836aa468c758ec65d9
Parents: 266a62a
Author: Roger Shi 
Authored: Wed Apr 5 18:35:55 2017 +0800
Committer: Roger Shi 
Committed: Wed Apr 5 18:36:16 2017 +0800

--
 .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 5 -
 1 file changed, 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/88e66f3e/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 67e811a..bbf3c60 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -19,12 +19,9 @@
 package org.apache.kylin.source.hive;
 
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.lang.IncompleteArgumentException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -61,8 +58,6 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Sets;
 
-import javax.el.MethodNotFoundException;
-
 public class HiveMRInput implements IMRInput {
 
 public static String getTableNameForHCat(TableDesc table, boolean 
isFullTable) {

[5/9] kylin git commit: KYLIN 2513 SUM measure return type should not overflow

2017-04-11 Thread liyang 
KYLIN 2513 SUM measure return type should not overflow


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/023fa238
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/023fa238
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/023fa238

Branch: refs/heads/master
Commit: 023fa2387f9685efc64cc21a0fcc7b2e4457d9e7
Parents: 88e66f3
Author: chenzhx <346839...@qq.com>
Authored: Thu Apr 6 15:34:14 2017 +0800
Committer: chenzhx <346839...@qq.com>
Committed: Thu Apr 6 15:34:14 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/023fa238/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 856e74f..73a6cd0 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -370,10 +370,10 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   
if(colType==="smallint"||colType==="int"||colType==="bigint"||colType==="integer"){
 $scope.newMeasure.function.returntype= 'bigint';
   }else{
-if(colType.indexOf('decimal')!=-1){
-  $scope.newMeasure.function.returntype= colType;
+   
if(colType.indexOf('decimal')!=-1||colType==="double"||colType==="float"){
+  $scope.newMeasure.function.returntype= 'decimal(19,4)';
 }else{
-  $scope.newMeasure.function.returntype= 'decimal';
+  $scope.newMeasure.function.returntype= 'decimal(14,0)';
 }
   }
   break;

[7/9] kylin git commit: minor, remove method fillbackOptimizedColumn

2017-04-11 Thread liyang 
minor, remove method fillbackOptimizedColumn


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b4f70d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b4f70d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b4f70d2

Branch: refs/heads/master
Commit: 6b4f70d257e1eb363a7b792cde8f6f59821094a6
Parents: 75630b8
Author: Roger Shi 
Authored: Sun Apr 9 21:26:09 2017 +0800
Committer: Roger Shi 
Committed: Sun Apr 9 21:26:09 2017 +0800

--
 .../kylin/query/relnode/OLAPAggregateRel.java | 18 --
 1 file changed, 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6b4f70d2/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 2c75a14..1378681 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -345,8 +345,6 @@ public class OLAPAggregateRel extends Aggregate implements 
OLAPRel {
 }
 
 private void buildRewriteFieldsAndMetricsColumns() {
-fillbackOptimizedColumn();
-
 ColumnRowType inputColumnRowType = ((OLAPRel) 
getInput()).getColumnRowType();
 RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
 for (int i = 0; i < this.aggregations.size(); i++) {
@@ -385,22 +383,6 @@ public class OLAPAggregateRel extends Aggregate implements 
OLAPRel {
 }
 }
 
-private void fillbackOptimizedColumn() {
-// some aggcall will be optimized out in sub-query (e.g. tableau 
generated sql), we need to fill them back
-RelDataType inputAggRow = getInput().getRowType();
-RelDataType outputAggRow = getRowType();
-if (inputAggRow.getFieldCount() != outputAggRow.getFieldCount()) {
-for (RelDataTypeField inputField : inputAggRow.getFieldList()) {
-String inputFieldName = inputField.getName();
-// constant columns(starts with $) should not be added to 
context.
-if (!inputFieldName.startsWith("$") && 
outputAggRow.getField(inputFieldName, true, false) == null) {
-TblColRef column = 
this.columnRowType.getColumnByIndex(inputField.getIndex());
-this.context.metricsColumns.add(column);
-}
-}
-}
-}
-
 private boolean noPrecaculatedFieldsAvailable() {
 return !this.context.hasPrecalculatedFields() || 
!RewriteImplementor.needRewrite(this.context);
 }

[9/9] kylin git commit: Merge commit 'a36288142f30cad931881107872adb4ba0a574a3'

2017-04-11 Thread liyang 
Merge commit 'a36288142f30cad931881107872adb4ba0a574a3'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e9ec116c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e9ec116c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e9ec116c

Branch: refs/heads/master
Commit: e9ec116c247072603a5405ddba425fd349ba670b
Parents: f44f828 a362881
Author: Hongbin Ma 
Authored: Tue Apr 11 17:12:46 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 17:12:46 2017 +0800

--
 build/conf/kylin.properties |  1 +
 .../kylin/common/util/HiveCmdBuilder.java   | 16 +++-
 .../kylin/common/util/HiveCmdBuilderTest.java   |  1 +
 .../cube/model/CubeJoinedFlatTableDesc.java |  4 +-
 .../org/apache/kylin/job/JoinedFlatTable.java   | 34 +++
 .../kylin/job/engine/JobEngineConfig.java   |  2 +-
 .../apache/kylin/job/JoinedFlatTableTest.java   |  4 +-
 .../metadata/filter/CompareTupleFilter.java | 14 +++
 .../filter/FilterOptimizeTransformer.java   | 89 ++
 .../kylin/metadata/model/FunctionDesc.java  |  8 +-
 .../kylin/metadata/model/ParameterDesc.java | 58 ++--
 .../apache/kylin/metadata/model/TblColRef.java  | 10 ++
 .../gtrecord/GTCubeStorageQueryBase.java|  2 +-
 .../gtrecord/StorageResponseGTScatter.java  | 22 +++--
 .../engine/mr/common/HadoopStatusGetter.java| 15 +--
 examples/test_case_data/sandbox/hdfs-site.xml   |  4 +
 .../test_case_data/sandbox/kylin.properties |  1 +
 .../src/test/resources/query/sql/query104.sql   | 31 +++
 .../resources/query/sql_casewhen/query55.sql| 22 +
 .../query/sql_casewhen/query55.sql.disable  | 22 -
 .../resources/query/sql_subquery/query15.sql| 29 ++
 .../resources/query/sql_subquery/query16.sql| 38 
 .../kylin/query/relnode/OLAPAggregateRel.java   | 13 ++-
 .../kylin/query/relnode/OLAPFilterRel.java  | 25 ++---
 .../kylin/query/relnode/OLAPProjectRel.java | 12 +--
 .../kylin/query/relnode/OLAPTableScan.java  |  2 +-
 .../org/apache/kylin/query/udf/ConcatUDF.java   | 28 ++
 .../org/apache/kylin/rest/util/QueryUtil.java   | 15 ---
 .../apache/kylin/rest/util/QueryUtilTest.java   |  6 --
 .../apache/kylin/source/hive/HiveMRInput.java   | 97 +---
 .../kylin/storage/hbase/HBaseConnection.java|  7 +-
 .../hbase/util/DeployCoprocessorCLI.java|  1 +
 32 files changed, 425 insertions(+), 208 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --cc core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 7313630,5553d34..54c1e61
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@@ -19,8 -19,8 +19,9 @@@
  package org.apache.kylin.job;
  
  import java.io.File;
 +import java.lang.reflect.Method;
  import java.util.HashSet;
+ import java.util.Map;
  import java.util.Set;
  
  import javax.xml.parsers.DocumentBuilder;
@@@ -49,18 -46,18 +50,17 @@@ public class JoinedFlatTable 
  return storageDfsDir + "/" + flatDesc.getTableName();
  }
  
- public static String generateHiveSetStatements(JobEngineConfig 
engineConfig) {
 -public static String generateHiveInitStatements(
 -String flatTableDatabase, String kylinHiveFile, Map cubeOverrides) {
++public static String generateHiveInitStatements(String flatTableDatabase, 
String kylinHiveFile, Map cubeOverrides) {
+ 
  StringBuilder buffer = new StringBuilder();
  
+ buffer.append("USE ").append(flatTableDatabase).append(";\n");
  try {
- File hadoopPropertiesFile = new 
File(engineConfig.getHiveConfFilePath());
- 
- if (hadoopPropertiesFile.exists()) {
+ File file = new File(kylinHiveFile);
+ if (file.exists()) {
  DocumentBuilderFactory factory = 
DocumentBuilderFactory.newInstance();
- DocumentBuilder builder;
- Document doc;
- builder = factory.newDocumentBuilder();
- doc = builder.parse(hadoopPropertiesFile);
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document doc = builder.parse(file);
  NodeList nl = doc.getElementsByTagName("property");
  for (int i = 0; i < nl.getLength(); i++) {
  String name = 
doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
@@@ -101,24 -103,8 +106,19 @@@
  return ddl.toString();
  }

[8/9] kylin git commit: minor bug fix

2017-04-11 Thread liyang 
minor bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f44f8289
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f44f8289
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f44f8289

Branch: refs/heads/master
Commit: f44f828928aaecac3e7dafee69d9443b043b0d4f
Parents: 6b4f70d
Author: Hongbin Ma 
Authored: Wed Apr 5 14:07:33 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 14:54:21 2017 +0800

--
 .../main/java/org/apache/kylin/rest/service/ProjectService.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f44f8289/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index e6f546c..8c0cf7a 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -104,7 +104,7 @@ public class ProjectService extends BasicService {
 return projects.subList(coffset, coffset + climit);
 }
 
-@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or 
hasPermission(#project, 'ADMINISTRATION') or hasPermission(#cube, 
'MANAGEMENT')")
+@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or 
hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 
'MANAGEMENT')")
 public void deleteProject(String projectName, ProjectInstance project) 
throws IOException {
 getProjectManager().dropProject(projectName);

[3/9] kylin git commit: KYLIN-2531 convert old tableExt jackson to new one instead of removing it

2017-04-11 Thread liyang 
KYLIN-2531 convert old tableExt jackson to new one instead of removing it


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/266a62a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/266a62a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/266a62a5

Branch: refs/heads/master
Commit: 266a62a5ab6b0b39b08b79d3f3d4d1948e111247
Parents: 38fe432
Author: Cheng Wang 
Authored: Wed Apr 5 14:43:52 2017 +0800
Committer: Dong Li 
Committed: Wed Apr 5 14:46:12 2017 +0800

--
 .../apache/kylin/metadata/MetadataManager.java  | 64 +++-
 .../kylin/metadata/MetadataManagerTest.java | 41 +++--
 2 files changed, 85 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/266a62a5/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 9427ace..f17983f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -19,8 +19,10 @@
 package org.apache.kylin.metadata;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -30,8 +32,10 @@ import java.util.concurrent.ConcurrentMap;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.RawResource;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
 import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
@@ -47,6 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 /**
  * Serves (and caches) metadata for Kylin instance.
@@ -207,11 +212,7 @@ public class MetadataManager {
 result.setUuid(UUID.randomUUID().toString());
 result.setLastModified(0);
 result.init();
-try {
-saveTableExt(result);
-} catch (IOException ex) {
-logger.warn("Failed to save TableExt", ex);
-}
+srcTableExdMap.put(result.getName(), result);
 }
 return result;
 }
@@ -224,9 +225,14 @@ public class MetadataManager {
 tableExt.init();
 
 String path = tableExt.getResourcePath();
-getStore().putResource(path, tableExt, TABLE_EXT_SERIALIZER);
 
-srcTableExdMap.put(tableExt.getName(), tableExt);
+ResourceStore store = getStore();
+
+TableExtDesc t = store.getResource(path, TableExtDesc.class, 
TABLE_EXT_SERIALIZER);
+if (t != null && t.getName() == null)
+store.deleteResource(path);
+
+store.putResource(path, tableExt, TABLE_EXT_SERIALIZER);
 }
 
 public void removeTableExt(String tableName) throws IOException {
@@ -387,22 +393,52 @@ public class MetadataManager {
 private TableExtDesc reloadTableExtAt(String path) throws IOException {
 ResourceStore store = getStore();
 TableExtDesc t = store.getResource(path, TableExtDesc.class, 
TABLE_EXT_SERIALIZER);
+
 if (t == null) {
 return null;
 }
+
+// convert old tableExt json to new one
+if (t.getName() == null) {
+t = convertOldTableExtToNewer(path);
+}
+
 t.init();
 
-String name = t.getName();
+srcTableExdMap.putLocal(t.getName(), t);
+return t;
+}
 
-// remove old json
-if (name == null) {
-getStore().deleteResource(path);
-return null;
+private TableExtDesc convertOldTableExtToNewer(String path) throws 
IOException {
+Map attrs = Maps.newHashMap();
+
+ResourceStore store = getStore();
+RawResource res = store.getResource(path);
+
+InputStream is = res.inputStream;
+
+try {
+attrs.putAll(JsonUtil.readValue(is, HashMap.class));
+} finally {
+if (is != null)
+is.close();
 }
 
-srcTableExdMap.putLocal(name, t);
+String cardinality = 
attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY);
 
-

[2/9] kylin git commit: KYLIN-2532 make Hive flat step more extensible

2017-04-11 Thread liyang 
KYLIN-2532 make Hive flat step more extensible


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38fe432d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38fe432d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38fe432d

Branch: refs/heads/master
Commit: 38fe432d914d0bde8a0ddda50a6867ea42b5ccbf
Parents: d216e22
Author: Roger Shi 
Authored: Wed Apr 5 13:12:00 2017 +0800
Committer: Roger Shi 
Committed: Wed Apr 5 00:20:55 2017 -0500

--
 .../org/apache/kylin/common/KylinConfigBase.java  |  8 
 .../org/apache/kylin/job/JoinedFlatTable.java | 18 +-
 .../org/apache/kylin/source/hive/HiveMRInput.java |  5 +
 3 files changed, 30 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/38fe432d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 760b7da..4361242 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -448,6 +448,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.job.error-record-threshold", "0"));
 }
 
+public boolean isAdvancedFlatTableUsed() {
+return 
Boolean.parseBoolean(getOptional("kylin.job.use-advanced-flat-table", "false"));
+}
+
+public String getAdvancedFlatTableClass() {
+return getOptional("kylin.job.advanced-flat-table.class");
+}
+
 // 

 // SOURCE.HIVE
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/38fe432d/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 9ed563f..7313630 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -19,12 +19,15 @@
 package org.apache.kylin.job;
 
 import java.io.File;
+import java.lang.reflect.Method;
 import java.util.HashSet;
 import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
@@ -99,9 +102,22 @@ public class JoinedFlatTable {
 }
 
 public static String generateInsertDataStatement(IJoinedFlatTableDesc 
flatDesc, JobEngineConfig engineConfig) {
+final KylinConfig kylinConfig = ((CubeSegment) 
flatDesc.getSegment()).getConfig();
 StringBuilder sql = new StringBuilder();
+
+if (kylinConfig.isAdvancedFlatTableUsed()) {
+try {
+Class advancedFlatTable = 
Class.forName(kylinConfig.getAdvancedFlatTableClass());
+Method method = 
advancedFlatTable.getMethod("generateInsertDataStatement", 
IJoinedFlatTableDesc.class, JobEngineConfig.class);
+return (String) method.invoke(null, flatDesc, engineConfig);
+} catch (Exception e) {
+throw new RuntimeException(e);
+}
+}
+
 sql.append(generateHiveSetStatements(engineConfig));
-sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + 
generateSelectDataStatement(flatDesc) + ";").append("\n");
+sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + 
generateSelectDataStatement(flatDesc) + ";\n");
+
 return sql.toString();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/38fe432d/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index bbf3c60..67e811a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -19,9 +19,12 @@
 package

[6/9] kylin git commit: KYLIN 2534 column family select mistake

2017-04-11 Thread liyang 
KYLIN 2534 column family select mistake


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75630b8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75630b8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75630b8d

Branch: refs/heads/master
Commit: 75630b8d90a900550a55f4a931eedee4b9eb3d21
Parents: 023fa23
Author: chenzhx <346839...@qq.com>
Authored: Thu Apr 6 15:42:52 2017 +0800
Committer: chenzhx <346839...@qq.com>
Committed: Thu Apr 6 15:42:52 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 21 ++--
 1 file changed, 11 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/75630b8d/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 5e7de33..d3e2922 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -339,18 +339,19 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var allMeasureNames = $scope.getAllMeasureNames();
 var tmpColumnFamily = $scope.cubeMetaFrame.hbase_mapping.column_family;
 
-angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily,index1) {
-  angular.forEach(colFamily.columns[0].measure_refs, function 
(measureName, index2) {
-var allIndex = allMeasureNames.indexOf(measureName);
+for(var j=0;j<$scope.cubeMetaFrame.hbase_mapping.column_family.length; 
j++) {
+  for (var 
i=0;i<$scope.cubeMetaFrame.hbase_mapping.column_family[j].columns[0].measure_refs.length;
 i++){
+var allIndex = 
allMeasureNames.indexOf($scope.cubeMetaFrame.hbase_mapping.column_family[j].columns[0].measure_refs[i]);
 if (allIndex == -1) {
-  tmpColumnFamily[index1].columns[0].measure_refs.splice(index2, 1);
+  tmpColumnFamily[j].columns[0].measure_refs.splice(i, 1);
+  i--
 }
-
-if (tmpColumnFamily[index1].columns[0].measure_refs == 0) {
-  tmpColumnFamily.splice(index1, 1);
-}
-  });
-});
+  }
+  if (tmpColumnFamily[j].columns[0].measure_refs.length == 0) {
+tmpColumnFamily.splice(j, 1);
+j--
+  }
+}
 
 $scope.cubeMetaFrame.hbase_mapping.column_family = tmpColumnFamily;
   };

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-11 Thread lidong 
http://git-wip-us.apache.org/repos/asf/kylin/blob/20dc8398/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/20dc8398/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-11 Thread lidong 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20dc8398
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20dc8398
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20dc8398

Branch: refs/heads/yang22-hbase102
Commit: 20dc839836c033d3800b2bbc4b72cd55f0b94600
Parents: a943dda
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 16:23:03 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/20dc8398/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[4/4] kylin git commit: KYLIN-2233 Support HBase 1.0.2

2017-04-11 Thread lidong 
KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/294eb196
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/294eb196
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/294eb196

Branch: refs/heads/yang22-hbase102
Commit: 294eb196f9d3c20f6c11e164f19aabd783554239
Parents: 20dc839
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 16:26:49 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dc23a2f..099316a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of

[1/4] kylin git commit: KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel [Forced Update!]

2017-04-11 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 2206543ec -> 294eb196f (forced update)


KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a943ddaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a943ddaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a943ddaa

Branch: refs/heads/yang22-hbase102
Commit: a943ddaa25a9e93fc30f0b2681f3463af3dee357
Parents: c2cdce8
Author: Hongbin Ma 
Authored: Tue Apr 11 14:35:38 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 14:35:38 2017 +0800

--
 .../java/org/apache/kylin/query/relnode/OLAPFilterRel.java   | 8 +++-
 1 file changed, 3 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a943ddaa/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index c826ff3..3306dbf 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -78,12 +78,10 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 private static class TupleFilterVisitor extends 
RexVisitorImpl {
 
 private final ColumnRowType inputRowType;
-private final OLAPContext context;
 
-public TupleFilterVisitor(ColumnRowType inputRowType, OLAPContext 
context) {
+public TupleFilterVisitor(ColumnRowType inputRowType) {
 super(true);
 this.inputRowType = inputRowType;
-this.context = context;
 }
 
 @Override
@@ -228,7 +226,6 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 @Override
 public TupleFilter visitInputRef(RexInputRef inputRef) {
 TblColRef column = 
inputRowType.getColumnByIndex(inputRef.getIndex());
-context.allColumns.add(column);
 ColumnTupleFilter filter = new ColumnTupleFilter(column);
 return filter;
 }
@@ -318,12 +315,13 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 return;
 }
 
-TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType, context);
+TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType);
 context.filter = this.condition.accept(visitor);
 // optimize the filter, the optimization has to be segment-irrelevant
 new FilterOptimizeTransformer().transform(context.filter);
 
 context.filterColumns = collectColumns(context.filter);
+context.allColumns.addAll(context.filterColumns);
 }
 
 private Set collectColumns(TupleFilter filter) {

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-11 Thread lidong 
http://git-wip-us.apache.org/repos/asf/kylin/blob/20dc8398/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/20dc8398/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2017-04-11 Thread lidong 
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c3177d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c3177d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c3177d3

Branch: refs/heads/yang22-cdh5.7
Commit: 0c3177d3c9e8f2e1fada61a86b1abc7c776ca33e
Parents: 20dc839
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Tue Apr 11 16:24:54 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c3177d3/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c3177d3/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c3177d3/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-11 Thread lidong 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20dc8398
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20dc8398
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20dc8398

Branch: refs/heads/yang22-cdh5.7
Commit: 20dc839836c033d3800b2bbc4b72cd55f0b94600
Parents: a943dda
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 16:23:03 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/20dc8398/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/4] kylin git commit: KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel [Forced Update!]

2017-04-11 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 014f72d83 -> 0c3177d3c (forced update)


KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a943ddaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a943ddaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a943ddaa

Branch: refs/heads/yang22-cdh5.7
Commit: a943ddaa25a9e93fc30f0b2681f3463af3dee357
Parents: c2cdce8
Author: Hongbin Ma 
Authored: Tue Apr 11 14:35:38 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 14:35:38 2017 +0800

--
 .../java/org/apache/kylin/query/relnode/OLAPFilterRel.java   | 8 +++-
 1 file changed, 3 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a943ddaa/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index c826ff3..3306dbf 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -78,12 +78,10 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 private static class TupleFilterVisitor extends 
RexVisitorImpl {
 
 private final ColumnRowType inputRowType;
-private final OLAPContext context;
 
-public TupleFilterVisitor(ColumnRowType inputRowType, OLAPContext 
context) {
+public TupleFilterVisitor(ColumnRowType inputRowType) {
 super(true);
 this.inputRowType = inputRowType;
-this.context = context;
 }
 
 @Override
@@ -228,7 +226,6 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 @Override
 public TupleFilter visitInputRef(RexInputRef inputRef) {
 TblColRef column = 
inputRowType.getColumnByIndex(inputRef.getIndex());
-context.allColumns.add(column);
 ColumnTupleFilter filter = new ColumnTupleFilter(column);
 return filter;
 }
@@ -318,12 +315,13 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 return;
 }
 
-TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType, context);
+TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType);
 context.filter = this.condition.accept(visitor);
 // optimize the filter, the optimization has to be segment-irrelevant
 new FilterOptimizeTransformer().transform(context.filter);
 
 context.filterColumns = collectColumns(context.filter);
+context.allColumns.addAll(context.filterColumns);
 }
 
 private Set collectColumns(TupleFilter filter) {

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-04-11 Thread lidong 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20dc8398
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20dc8398
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20dc8398

Branch: refs/heads/yang22-hbase1.x
Commit: 20dc839836c033d3800b2bbc4b72cd55f0b94600
Parents: a943dda
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Tue Apr 11 16:23:03 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/20dc8398/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/3] kylin git commit: KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel [Forced Update!]

2017-04-11 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 8ce546fe9 -> 20dc83983 (forced update)


KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a943ddaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a943ddaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a943ddaa

Branch: refs/heads/yang22-hbase1.x
Commit: a943ddaa25a9e93fc30f0b2681f3463af3dee357
Parents: c2cdce8
Author: Hongbin Ma 
Authored: Tue Apr 11 14:35:38 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 14:35:38 2017 +0800

--
 .../java/org/apache/kylin/query/relnode/OLAPFilterRel.java   | 8 +++-
 1 file changed, 3 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a943ddaa/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index c826ff3..3306dbf 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -78,12 +78,10 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 private static class TupleFilterVisitor extends 
RexVisitorImpl {
 
 private final ColumnRowType inputRowType;
-private final OLAPContext context;
 
-public TupleFilterVisitor(ColumnRowType inputRowType, OLAPContext 
context) {
+public TupleFilterVisitor(ColumnRowType inputRowType) {
 super(true);
 this.inputRowType = inputRowType;
-this.context = context;
 }
 
 @Override
@@ -228,7 +226,6 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 @Override
 public TupleFilter visitInputRef(RexInputRef inputRef) {
 TblColRef column = 
inputRowType.getColumnByIndex(inputRef.getIndex());
-context.allColumns.add(column);
 ColumnTupleFilter filter = new ColumnTupleFilter(column);
 return filter;
 }
@@ -318,12 +315,13 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 return;
 }
 
-TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType, context);
+TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType);
 context.filter = this.condition.accept(visitor);
 // optimize the filter, the optimization has to be segment-irrelevant
 new FilterOptimizeTransformer().transform(context.filter);
 
 context.filterColumns = collectColumns(context.filter);
+context.allColumns.addAll(context.filterColumns);
 }
 
 private Set collectColumns(TupleFilter filter) {

kylin git commit: KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel

2017-04-11 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/yang22 c2cdce8a3 -> a943ddaa2


KYLIN-2539 move FilterOptimizeTransformer to OLAPFilterRel


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a943ddaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a943ddaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a943ddaa

Branch: refs/heads/yang22
Commit: a943ddaa25a9e93fc30f0b2681f3463af3dee357
Parents: c2cdce8
Author: Hongbin Ma 
Authored: Tue Apr 11 14:35:38 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 11 14:35:38 2017 +0800

--
 .../java/org/apache/kylin/query/relnode/OLAPFilterRel.java   | 8 +++-
 1 file changed, 3 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a943ddaa/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index c826ff3..3306dbf 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -78,12 +78,10 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 private static class TupleFilterVisitor extends 
RexVisitorImpl {
 
 private final ColumnRowType inputRowType;
-private final OLAPContext context;
 
-public TupleFilterVisitor(ColumnRowType inputRowType, OLAPContext 
context) {
+public TupleFilterVisitor(ColumnRowType inputRowType) {
 super(true);
 this.inputRowType = inputRowType;
-this.context = context;
 }
 
 @Override
@@ -228,7 +226,6 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 @Override
 public TupleFilter visitInputRef(RexInputRef inputRef) {
 TblColRef column = 
inputRowType.getColumnByIndex(inputRef.getIndex());
-context.allColumns.add(column);
 ColumnTupleFilter filter = new ColumnTupleFilter(column);
 return filter;
 }
@@ -318,12 +315,13 @@ public class OLAPFilterRel extends Filter implements 
OLAPRel {
 return;
 }
 
-TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType, context);
+TupleFilterVisitor visitor = new 
TupleFilterVisitor(this.columnRowType);
 context.filter = this.condition.accept(visitor);
 // optimize the filter, the optimization has to be segment-irrelevant
 new FilterOptimizeTransformer().transform(context.filter);
 
 context.filterColumns = collectColumns(context.filter);
+context.allColumns.addAll(context.filterColumns);
 }
 
 private Set collectColumns(TupleFilter filter) {