date:20170209

kylin git commit: KYLIN-2437 collect number of bytes scanned to query metrics

2017-02-09 Thread gaodayue

Repository: kylin
Updated Branches:
  refs/heads/master ecf6a69fe -> e09338b34


KYLIN-2437 collect number of bytes scanned to query metrics


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e09338b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e09338b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e09338b3

Branch: refs/heads/master
Commit: e09338b34c0b07a7167096e45bf9185aa0d0cbd5
Parents: ecf6a69
Author: gaodayue 
Authored: Wed Feb 8 13:59:31 2017 +0800
Committer: gaodayue 
Committed: Fri Feb 10 15:39:11 2017 +0800

--
 .../kylin/gridtable/GTAggregateScanner.java |  11 +-
 .../apache/kylin/gridtable/GTScanRequest.java   |  10 +-
 .../apache/kylin/gridtable/ScannerWorker.java   |  64 -
 .../apache/kylin/storage/StorageContext.java|   9 ++
 .../storage/gtrecord/CubeSegmentScanner.java|   3 +-
 .../kylin/storage/gtrecord/ScannerWorker.java   |  71 ++
 .../apache/kylin/rest/response/SQLResponse.java |  10 ++
 .../apache/kylin/rest/service/QueryService.java |   9 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   7 +-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |   6 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  17 ++-
 .../coprocessor/endpoint/CubeVisitService.java  | 142 +++
 .../endpoint/generated/CubeVisitProtos.java | 107 --
 .../endpoint/protobuf/CubeVisit.proto   |   1 +
 14 files changed, 309 insertions(+), 158 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 147dbc1..dd359f8 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -65,7 +65,6 @@ public class GTAggregateScanner implements IGTScanner {
 final AggregationCache aggrCache;
 final long spillThreshold; // 0 means no memory control && no spill
 final int storagePushDownLimit;//default to be Int.MAX
-final long deadline;
 final boolean spillEnabled;
 
 private int aggregatedRowCount = 0;
@@ -73,10 +72,10 @@ public class GTAggregateScanner implements IGTScanner {
 private boolean[] aggrMask;
 
 public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req) {
-this(inputScanner, req, Long.MAX_VALUE, true);
+this(inputScanner, req, true);
 }
 
-public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req, long 
deadline, boolean spillEnabled) {
+public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req, 
boolean spillEnabled) {
 if (!req.hasAggregation())
 throw new IllegalStateException();
 
@@ -90,7 +89,6 @@ public class GTAggregateScanner implements IGTScanner {
 this.spillThreshold = (long) (req.getAggCacheMemThreshold() * 
MemoryBudgetController.ONE_GB);
 this.aggrMask = new boolean[metricsAggrFuncs.length];
 this.storagePushDownLimit = req.getStoragePushDownLimit();
-this.deadline = deadline;
 this.spillEnabled = spillEnabled;
 
 Arrays.fill(aggrMask, true);
@@ -145,11 +143,6 @@ public class GTAggregateScanner implements IGTScanner {
 long count = 0;
 for (GTRecord r : inputScanner) {
 
-//check deadline
-if (count % GTScanRequest.terminateCheckInterval == 1 && 
System.currentTimeMillis() > deadline) {
-throw new GTScanTimeoutException("Timeout in 
GTAggregateScanner with scanned count " + count);
-}
-
 if (getNumOfSpills() == 0) {
 //check limit
 boolean ret = aggrCache.aggregate(r, storagePushDownLimit);

http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 7c94f5a..651e5c4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -156,7 +156,7 @@ public class GTScanRequest {
 }
 
 public IGTScanner decorateScanner(IGTScanner scanner) throws IOException {
-return decorateScanner(scanner, true, true, Long.MAX_VALUE);
+return

kylin git commit: KYLIN-2428 try fix guava in Spark IT again

2017-02-09 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 5f4c02f89 -> e40832dc5


KYLIN-2428 try fix guava in Spark IT again


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e40832dc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e40832dc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e40832dc

Branch: refs/heads/KYLIN-2428
Commit: e40832dc56c465059f087eabecbc0c77e51cf200
Parents: 5f4c02f
Author: Billy Liu 
Authored: Fri Feb 10 14:35:03 2017 +0800
Committer: Billy Liu 
Committed: Fri Feb 10 14:35:03 2017 +0800

--
 .../apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java  | 1 +
 kylin-it/pom.xml| 5 -
 2 files changed, 1 insertion(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e40832dc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 327d215..dd37c26 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -60,6 +60,7 @@ public class SparkBatchCubingJobBuilder2 extends 
BatchCubingJobBuilder2 {
 StringUtil.appendWithSeparator(jars, 
findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar
 StringUtil.appendWithSeparator(jars, 
findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar
 StringUtil.appendWithSeparator(jars, 
findJar("com.yammer.metrics.core.Gauge")); // metrics-core.jar
+StringUtil.appendWithSeparator(jars, 
findJar("com.google.common.base.Preconditions")); // guava.jar
 
 StringUtil.appendWithSeparator(jars, 
seg.getConfig().getSparkAdditionalJars());
 sparkExecutable.setJars(jars.toString());

http://git-wip-us.apache.org/repos/asf/kylin/blob/e40832dc/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index e62eb76..91104ba 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -258,11 +258,6 @@
 spark-hive_2.10
 provided
 
-
-com.google.guava
-guava
-provided
-

kylin git commit: KYLIN-2428 fix spark dep in IT

2017-02-09 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 8030a2727 -> 5f4c02f89


KYLIN-2428 fix spark dep in IT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f4c02f8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f4c02f8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f4c02f8

Branch: refs/heads/KYLIN-2428
Commit: 5f4c02f8944a53352fce764ee85e44bd148db2fd
Parents: 8030a27
Author: Billy Liu 
Authored: Fri Feb 10 14:06:16 2017 +0800
Committer: Billy Liu 
Committed: Fri Feb 10 14:06:16 2017 +0800

--
 engine-spark/pom.xml | 39 ---
 kylin-it/pom.xml |  5 +
 2 files changed, 5 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f4c02f8/engine-spark/pom.xml
--
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index fe6d998..799ea43 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -117,45 +117,6 @@
 maven-model
 test
 
-
-com.google.guava
-guava
-
-
-
 
 
-
-
-
-org.apache.maven.plugins
-maven-shade-plugin
-
-
-package
-
-shade
-
-
-false
-
true
-job
-
-
-*:*
-
-META-INF/*.SF
-META-INF/*.DSA
-META-INF/*.RSA
-
-
-
-
-
-
-
-
-
-
-
 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f4c02f8/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 91104ba..e62eb76 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -258,6 +258,11 @@
 spark-hive_2.10
 provided
 
+
+com.google.guava
+guava
+provided
+

kylin git commit: KYLIN-2441 protocol for REST API result format

2017-02-09 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master f2e8b690f -> 41332f3ed


KYLIN-2441 protocol for REST API result format


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41332f3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41332f3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41332f3e

Branch: refs/heads/master
Commit: 41332f3ed430b095efe5edefc831f0ec27f4b2ee
Parents: f2e8b69
Author: Hongbin Ma 
Authored: Fri Feb 10 12:34:49 2017 +0800
Committer: Hongbin Ma 
Committed: Fri Feb 10 12:35:14 2017 +0800

--
 .../rest/controller/EncodingController.java |  3 +-
 .../rest/exception/BadRequestException.java | 37 +---
 .../kylin/rest/response/EnvelopeResponse.java   | 34 --
 .../kylin/rest/response/ErrorResponse.java  | 32 -
 .../kylin/rest/response/ResponseCode.java   | 30 
 5 files changed, 79 insertions(+), 57 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/41332f3e/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
index b95394c..4a8b122 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
@@ -24,6 +24,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.response.ResponseCode;
 import org.apache.kylin.rest.service.EncodingService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,6 +66,6 @@ public class EncodingController extends BasicController {
 datatypeValidEncodings.put(dataTypeStr, 
encodingService.getValidEncodings(DataType.getType(dataTypeStr)));
 }
 
-return new EnvelopeResponse(EnvelopeResponse.CODE_SUCCESS, 
datatypeValidEncodings, "");
+return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, 
datatypeValidEncodings, "");
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/41332f3e/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
 
b/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
index 42b671b..af1995b 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.rest.exception;
 
+import org.apache.kylin.rest.response.ResponseCode;
 import org.springframework.http.HttpStatus;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
@@ -27,35 +28,31 @@ import 
org.springframework.web.bind.annotation.ResponseStatus;
  */
 @ResponseStatus(value = HttpStatus.BAD_REQUEST)
 public class BadRequestException extends RuntimeException {
-/**
- * 
- */
+
 private static final long serialVersionUID = -6798154278095441848L;
 
-public BadRequestException(String s) {
-super(s);
-}
+private String code;
 
 /**
- * 
+ * legacy support, new APIs should not call this. Instead, new APIs should 
provide return code
  */
-public BadRequestException() {
-super();
+public BadRequestException(String msg) {
+super(msg);
+this.code = ResponseCode.CODE_UNDEFINED;
 }
 
-/**
- * @param arg0
- * @param arg1
- */
-public BadRequestException(String arg0, Throwable arg1) {
-super(arg0, arg1);
+public BadRequestException(String msg, String code) {
+super(msg);
+this.code = code;
 }
 
-/**
- * @param arg0
- */
-public BadRequestException(Throwable arg0) {
-super(arg0);
+public BadRequestException(String msg, String code, Throwable cause) {
+super(msg, cause);
+this.code = code;
+}
+
+public String getCode() {
+return code;
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/41332f3e/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java

[4/9] kylin git commit: KYLIN-2440 Make PK/FK always exposed

2017-02-09 Thread lidong

KYLIN-2440 Make PK/FK always exposed


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7fa25e0f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7fa25e0f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7fa25e0f

Branch: refs/heads/master-hbase0.98
Commit: 7fa25e0fafe25686450587407f2ed6f1e71cf4d3
Parents: f456961
Author: Li Yang 
Authored: Thu Feb 9 18:28:38 2017 +0800
Committer: Li Yang 
Committed: Thu Feb 9 18:28:53 2017 +0800

--
 .../org/apache/kylin/cube/model/CubeDesc.java   | 35 +---
 1 file changed, 31 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7fa25e0f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5e970bf..9acb90c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -64,6 +64,7 @@ import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
@@ -570,10 +571,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey 
columns count (%d) doesn't match dimensions columns count (%d)", 
rowkey.getRowKeyColumns().length, dimCols.size());
 
 initDictionaryDesc();
-
-for (TblColRef col : allColumns) {
-allColumnDescs.add(col.getColumnDesc());
-}
+amendAllColumns();
 }
 
 public void validateAggregationGroups() {
@@ -958,6 +956,35 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 return false;
 }
 
+private void amendAllColumns() {
+// make sure all PF/FK are included, thus become exposed to calcite 
later
+Set tables = collectTablesOnJoinChain(allColumns);
+for (TableRef t : tables) {
+JoinDesc join = model.getJoinByPKSide(t);
+if (join != null) {
+allColumns.addAll(Arrays.asList(join.getForeignKeyColumns()));
+allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns()));
+}
+}
+
+for (TblColRef col : allColumns) {
+allColumnDescs.add(col.getColumnDesc());
+}
+}
+
+private Set collectTablesOnJoinChain(Set columns) {
+Set result = new HashSet<>();
+for (TblColRef col : columns) {
+TableRef t = col.getTableRef();
+while (t != null) {
+result.add(t);
+JoinDesc join = model.getJoinByPKSide(t);
+t = join == null ? null : join.getFKSide();
+}
+}
+return result;
+}
+
 public long getRetentionRange() {
 return retentionRange;
 }

[1/9] kylin git commit: KYLIN-2308 bug fix and improve user experience [Forced Update!]

2017-02-09 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master-hbase0.98 39c8ac30e -> d6a844bb1 (forced update)


KYLIN-2308 bug fix and improve user experience


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e9f0908
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e9f0908
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e9f0908

Branch: refs/heads/master-hbase0.98
Commit: 1e9f0908b9942113bbd251dc5eeba71973cc77e1
Parents: 7742b31
Author: kangkaisen 
Authored: Thu Feb 9 12:06:56 2017 +0800
Committer: kangkaisen 
Committed: Thu Feb 9 15:54:01 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 36 ++--
 webapp/app/js/controllers/cubeSchema.js |  8 +
 .../cubeDesigner/advanced_settings.html |  2 +-
 3 files changed, 43 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 39d36b0..7cdb1cd 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -342,12 +342,44 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 return assignedMeasures;
   };
 
-  if ($scope.getAllMeasureNames().length != 
$scope.getAssignedMeasureNames().length) {
+  $scope.rmDeprecatedMeasureNames = function () {
+var allMeasureNames = $scope.getAllMeasureNames();
+var tmpColumnFamily = $scope.cubeMetaFrame.hbase_mapping.column_family;
+
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily,index1) {
+  angular.forEach(colFamily.columns[0].measure_refs, function 
(measureName, index2) {
+var allIndex = allMeasureNames.indexOf(measureName);
+if (allIndex == -1) {
+  tmpColumnFamily[index1].columns[0].measure_refs.splice(index2, 1);
+}
+
+if (tmpColumnFamily[index1].columns[0].measure_refs == 0) {
+  tmpColumnFamily.splice(index1, 1);
+}
+  });
+});
+
+$scope.cubeMetaFrame.hbase_mapping.column_family = tmpColumnFamily;
+  };
+
+  if ($scope.getAssignedMeasureNames().length == 0) {
 $scope.initColumnFamily();
+  } else {
+$scope.rmDeprecatedMeasureNames();
+if ($scope.getAllMeasureNames().length > 
$scope.getAssignedMeasureNames().length) {
+  $scope.initColumnFamily();
+}
   }
 
-
   $scope.addColumnFamily = function () {
+var isCFEmpty = _.some($scope.cubeMetaFrame.hbase_mapping.column_family, 
function(colFamily) {
+  return colFamily.columns[0].measure_refs.length == 0;
+});
+
+if (isCFEmpty === true) {
+  return;
+}
+
 var colFamily = 
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 
1);
 $scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
   };

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/js/controllers/cubeSchema.js
--
diff --git a/webapp/app/js/controllers/cubeSchema.js 
b/webapp/app/js/controllers/cubeSchema.js
index 17371f2..4beeebe 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -287,6 +287,14 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, 
QueryService, UserServic
   errors.push("All measures need to be assigned to column family");
 }
 
+var isCFEmpty = 
_.some($scope.cubeMetaFrame.hbase_mapping.column_family, function(colFamily) {
+  return colFamily.columns[0].measure_refs.length == 0;
+});
+
+if (isCFEmpty == true) {
+  errors.push("Each column family can't not be empty");
+}
+
 
 var errorInfo = "";
 angular.forEach(errors,function(item){

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html 
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index bf95256..f26722b 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -453,7 +453,7 @@
 
  

-   
+

[7/9] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

2017-02-09 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/d6a844bb/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
--
diff --git 
a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.ResourceStore;
@@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI {
 Serializer projectSerializer = new 
JsonSerializer(ProjectInstance.class);
 ProjectInstance project = store.getResource(projectResPath, 
ProjectInstance.class, projectSerializer);
 String projUUID = project.getUuid();
-Table aclHtable = null;
+HTableInterface aclHtable = null;
 try {
-aclHtable = 
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix()
 + "_acl"));
+aclHtable = 
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix()
 + "_acl");
 
 // cube acl
 Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI {
 aclHtable.put(put);
 }
 }
+aclHtable.flushCommits();
 } finally {
 IOUtils.closeQuietly(aclHtable);
 }

[3/9] kylin git commit: minor, decorateScanner can skip filter only

2017-02-09 Thread lidong

minor, decorateScanner can skip filter only

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f4569619
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f4569619
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f4569619

Branch: refs/heads/master-hbase0.98
Commit: f456961986e5c6ab94080a0fa74fa86c1a33c273
Parents: abf5332
Author: Roger Shi 
Authored: Thu Feb 9 17:14:57 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Feb 9 17:35:05 2017 +0800

--
 .../main/java/org/apache/kylin/gridtable/GTScanRequest.java | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f4569619/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index c45f90c..7976e3f 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -166,13 +166,20 @@ public class GTScanRequest {
  * Refer to CoprocessorBehavior for explanation
  */
 public IGTScanner decorateScanner(IGTScanner scanner, boolean 
filterToggledOn, boolean aggrToggledOn, long deadline) throws IOException {
+return decorateScanner(scanner, filterToggledOn, aggrToggledOn, false, 
deadline);
+}
+
+/**
+ * hasPreFiltered indicate the data has been filtered before scanning
+ */
+public IGTScanner decorateScanner(IGTScanner scanner, boolean 
filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, long deadline) 
throws IOException {
 IGTScanner result = scanner;
 if (!filterToggledOn) { //Skip reading this section if you're not 
profiling! 
 int scanned = lookAndForget(result);
 return new EmptyGTScanner(scanned);
 } else {
 
-if (this.hasFilterPushDown()) {
+if (this.hasFilterPushDown() && !hasPreFiltered) {
 result = new GTFilterScanner(result, this);
 }

[6/9] kylin git commit: fix some testcases which can not be run concurrently

2017-02-09 Thread lidong

fix some testcases which can not be run concurrently

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2e8b690
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2e8b690
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2e8b690

Branch: refs/heads/master-hbase0.98
Commit: f2e8b690f186abe5048dee8a6b0339c2a28c0594
Parents: 3ee4946
Author: etherge 
Authored: Wed Feb 8 23:50:35 2017 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 22:05:11 2017 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   | 16 +++---
 .../apache/kylin/common/KylinConfigBase.java|  7 ++-
 .../apache/kylin/common/KylinConfigTest.java| 17 ++-
 .../apache/kylin/dict/CachedTreeMapTest.java|  7 ++-
 .../kylin/job/BaseTestDistributedScheduler.java | 29 ++-
 .../kylin/provision/BuildCubeWithStream.java| 32 ++--
 .../org/apache/kylin/provision/MockKafka.java   | 11 ++---
 .../hbase/util/ZookeeperDistributedJobLock.java | 24 +
 .../kylin/storage/hbase/util/ZookeeperUtil.java | 52 
 9 files changed, 124 insertions(+), 71 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index e8c7fae..fdcd52c 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -56,6 +56,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
+import com.google.common.io.Files;
 
 public class DeployUtil {
 private static final Logger logger = 
LoggerFactory.getLogger(DeployUtil.class);
@@ -139,7 +140,7 @@ public class DeployUtil {
 boolean buildCubeUsingProvidedData = 
Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData"));
 if (!buildCubeUsingProvidedData) {
 System.out.println("build cube with random dataset");
-
+
 // data is generated according to cube descriptor and saved in 
resource store
 MetadataManager mgr = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
 DataModelDesc model = mgr.getDataModelDesc(modelName);
@@ -208,12 +209,12 @@ public class DeployUtil {
 MetadataManager metaMgr = MetadataManager.getInstance(config());
 
 // scp data files, use the data from hbase, instead of local files
-File temp = File.createTempFile("temp", ".csv");
-temp.createNewFile();
+File tempDir = Files.createTempDir();
+String tempDirAbsPath = tempDir.getAbsolutePath();
 for (String tablename : TABLE_NAMES) {
 tablename = tablename.toUpperCase();
 
-File localBufferFile = new File(temp.getParent() + "/" + tablename 
+ ".csv");
+File localBufferFile = new File(tempDirAbsPath + "/" + tablename + 
".csv");
 localBufferFile.createNewFile();
 
 InputStream hbaseDataStream = 
metaMgr.getStore().getResource("/data/" + tablename + ".csv").inputStream;
@@ -225,8 +226,7 @@ public class DeployUtil {
 
 localBufferFile.deleteOnExit();
 }
-String tableFileDir = temp.getParent();
-temp.delete();
+tempDir.deleteOnExit();
 
 IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
@@ -238,7 +238,7 @@ public class DeployUtil {
 // load data to hive tables
 // LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename
 for (String tablename : TABLE_NAMES) {
-hiveClient.executeHQL(generateLoadDataHql(tablename.toUpperCase(), 
tableFileDir));
+hiveClient.executeHQL(generateLoadDataHql(tablename.toUpperCase(), 
tempDirAbsPath));
 }
 
 final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
@@ -255,7 +255,7 @@ public class DeployUtil {
 
 String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity();
 String dropsql2 = "DROP VIEW IF EXISTS " + tableDesc.getIdentity();
-
+
 StringBuilder ddl = new StringBuilder();
 ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
 ddl.append("(" + "\n");

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java

[8/9] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

2017-02-09 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/d6a844bb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 74ab017..a44de4f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -30,15 +30,14 @@ import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +68,7 @@ public class HBaseResourceStore extends ResourceStore {
 final String tableNameBase;
 final String hbaseUrl;
 
-Connection getConnection() throws IOException {
+HConnection getConnection() throws IOException {
 return HBaseConnection.get(hbaseUrl);
 }
 
@@ -121,7 +120,7 @@ public class HBaseResourceStore extends ResourceStore {
 byte[] endRow = Bytes.toBytes(lookForPrefix);
 endRow[endRow.length - 1]++;
 
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 Scan scan = new Scan(startRow, endRow);
 if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
 scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,12 +237,13 @@ public class HBaseResourceStore extends ResourceStore {
 IOUtils.copy(content, bout);
 bout.close();
 
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 byte[] row = Bytes.toBytes(resPath);
 Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
 
 table.put(put);
+table.flushCommits();
 } finally {
 IOUtils.closeQuietly(table);
 }
@@ -251,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected long checkAndPutResourceImpl(String resPath, byte[] content, 
long oldTS, long newTS) throws IOException, IllegalStateException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 byte[] row = Bytes.toBytes(resPath);
 byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -264,6 +264,8 @@ public class HBaseResourceStore extends ResourceStore {
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);
 }
 
+table.flushCommits();
+
 return newTS;
 } finally {
 IOUtils.closeQuietly(table);
@@ -272,7 +274,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected void deleteResourceImpl(String resPath) throws IOException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 boolean hdfsResourceExist = false;
 Result result = internalGetFromHTable(table, resPath, true, false);
@@ -285,6 +287,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 Delete del = new Delete(Bytes.toBytes(resPath));
 table.delete(del);
+table.flushCommits();
 
 if (hdfsResourceExist) { // remove hdfs cell value
 Path redirectPath = bigCellHDFSPath(resPath);
@@ -305,7 +308,7 @@ public class HBaseResourceStore extends ResourceStore {
 }
 
 private Result getFromHTable(String path, boolean fetchContent, boolean 
fetchTimestamp) throws IOException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =

kylin git commit: KYLIN-2428 fix shade

2017-02-09 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 e9024fbb4 -> bce09a881


KYLIN-2428 fix shade


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bce09a88
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bce09a88
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bce09a88

Branch: refs/heads/KYLIN-2428
Commit: bce09a881beab57fe2609e37b5a52df0a5a1d048
Parents: e9024fb
Author: Billy Liu 
Authored: Thu Feb 9 23:02:26 2017 +0800
Committer: Billy Liu 
Committed: Thu Feb 9 23:02:26 2017 +0800

--
 assembly/pom.xml | 44 ++--
 tool/pom.xml | 12 
 2 files changed, 46 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bce09a88/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 7d895fc..e8d933e 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -188,16 +188,48 @@
 
 
 
-com
-
${shadeBase}.com
+com.fasterxml.jackson
+
${shadeBase}.com.fasterxml.jackson
 
 
-javax
-
${shadeBase}.javax
+com.sun.activation
+
${shadeBase}.com.sun.activation
 
 
-org.apache.commons
-
${shadeBase}.org.apache.commons
+com.tdunning
+
${shadeBase}.com.tdunning
+
+
+com.fasterxml.jackson
+
${shadeBase}.com.fasterxml.jackson
+
+
+javax.activation
+
${shadeBase}.javax.activation
+
+
+javax.mail
+
${shadeBase}.javax.mail
+
+
+org.apache.commons.codec
+
${shadeBase}.org.apache.commons.codec
+
+
+
org.apache.commons.collections
+
${shadeBase}.org.apache.commons.collections
+
+
+org.apache.commons.lang3
+
${shadeBase}.org.apache.commons.lang3
+
+
+
org.apache.commons.logging
+
${shadeBase}.org.apache.commons.logging
+
+
+org.apache.commons.mail
+
${shadeBase}.org.apache.commons.mail
 
 
 org.roaringbitmap

http://git-wip-us.apache.org/repos/asf/kylin/blob/bce09a88/tool/pom.xml
--
diff --git a/tool/pom.xml b/tool/pom.xml
index 70293d0..6c1df4f 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -142,12 +142,16 @@
 
 
 
-com
-
${shadeBase}.com
+com.google.common
+
${shadeBase}.com.google.common
 
 
-org.apache.commons
-
${shadeBase}.org.apache.commons
+org.apache.commons.io
+
${shadeBase}.org.apache.commons.io
+
+
+org.apache.commons.lang
+

kylin git commit: KYLIN-2428 shade third party libraries in job/coprocessor/tool

2017-02-09 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 f2f0c7eb0 -> e9024fbb4


KYLIN-2428 shade third party libraries in job/coprocessor/tool


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e9024fbb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e9024fbb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e9024fbb

Branch: refs/heads/KYLIN-2428
Commit: e9024fbb4c2c27335a77d54d0fedf7d1255de6db
Parents: f2f0c7e
Author: Billy Liu 
Authored: Thu Feb 9 21:58:20 2017 +0800
Committer: Billy Liu 
Committed: Thu Feb 9 21:58:20 2017 +0800

--
 assembly/pom.xml | 30 ++---
 build/bin/get-properties.sh  |  2 +-
 build/conf/kylin-tools-init-log4j.properties | 32 ---
 build/conf/kylin-tools-log4j.properties  |  9 ---
 storage-hbase/pom.xml| 15 ++-
 tool/pom.xml | 22 
 6 files changed, 68 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e9024fbb/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 873abff..7d895fc 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -19,16 +19,20 @@
 
 http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+kylin-assembly
+Apache Kylin - Assembly
+
 
 kylin
 org.apache.kylin
 2.0.0-SNAPSHOT
-
 
-4.0.0
 
-kylin-assembly
-Apache Kylin - Assembly
+
+org.apache.kylin.job.shaded
+
 
 
 
@@ -182,6 +186,24 @@
 
org.apache.httpcomponents:*
 
 
+
+
+com
+
${shadeBase}.com
+
+
+javax
+
${shadeBase}.javax
+
+
+org.apache.commons
+
${shadeBase}.org.apache.commons
+
+
+org.roaringbitmap
+
${shadeBase}.org.roaringbitmap
+
+
 
 
 *:*

http://git-wip-us.apache.org/repos/asf/kylin/blob/e9024fbb/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 4be9740..d6ebc2e 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -27,5 +27,5 @@ fi
 
 job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
 tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-result=`java -cp $job_jar:$tool_jar 
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-init-log4j.properties 
org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
+result=`java -cp $job_jar:$tool_jar 
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties 
org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
 echo "$result"

http://git-wip-us.apache.org/repos/asf/kylin/blob/e9024fbb/build/conf/kylin-tools-init-log4j.properties
--
diff --git a/build/conf/kylin-tools-init-log4j.properties 
b/build/conf/kylin-tools-init-log4j.properties
deleted file mode 100644
index 650745d..000
--- a/build/conf/kylin-tools-init-log4j.properties
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#  
-# http://www.apache.org/licenses/LICENSE-2.0
-#  
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for

kylin git commit: KYLIN-2316 Global dict couldn't be used for dimension column

2017-02-09 Thread kangkaisen

Repository: kylin
Updated Branches:
  refs/heads/master 7fa25e0fa -> 3ee49467c


KYLIN-2316 Global dict couldn't be used for dimension column


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ee49467
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ee49467
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ee49467

Branch: refs/heads/master
Commit: 3ee49467ce686d11afcecbc636bb5213b0927427
Parents: 7fa25e0
Author: kangkaisen 
Authored: Thu Feb 9 19:54:26 2017 +0800
Committer: kangkaisen 
Committed: Thu Feb 9 20:28:17 2017 +0800

--
 .../kylin/cube/model/validation/rule/DictionaryRule.java |  9 +
 .../cube/model/validation/rule/DictionaryRuleTest.java   | 11 +--
 2 files changed, 18 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ee49467/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index 37889c2..8da3ca0 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -29,6 +29,7 @@ import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.validation.IValidatorRule;
 import org.apache.kylin.cube.model.validation.ResultLevel;
 import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.dict.GlobalDictionaryBuilder;
 import org.apache.kylin.metadata.model.TblColRef;
 
 /**
@@ -45,10 +46,13 @@ public class DictionaryRule implements 
IValidatorRule {
 static final String ERROR_REUSE_BUILDER_BOTH_SET = "REUSE and BUILDER both 
set on dictionary for column: ";
 static final String ERROR_REUSE_BUILDER_BOTH_EMPTY = "REUSE and BUILDER 
both empty on dictionary for column: ";
 static final String ERROR_TRANSITIVE_REUSE = "Transitive REUSE is not 
allowed for dictionary: ";
+static final String ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE = "Global 
dictionary couldn't be used for dimension column: ";
 
 @Override
 public void validate(CubeDesc cubeDesc, ValidateContext context) {
 List dictDescs = cubeDesc.getDictionaries();
+Set dimensionColumns = 
cubeDesc.listDimensionColumnsIncludingDerived();
+
 if (dictDescs == null || dictDescs.isEmpty()) {
 return;
 }
@@ -78,6 +82,11 @@ public class DictionaryRule implements 
IValidatorRule {
 return;
 }
 
+if (StringUtils.isNotEmpty(builderClass) && 
builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) && 
dimensionColumns.contains(dictCol)) {
+context.addResult(ResultLevel.ERROR, 
ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE + dictCol);
+return;
+}
+
 if (reuseCol != null) {
 reuseDictionaries.add(dictDesc);
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ee49467/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
 
b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index 452768f..fcb723e 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.cube.model.validation.rule;
 
 import static 
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_DUPLICATE_DICTIONARY_COLUMN;
+import static 
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE;
 import static 
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_EMPTY;
 import static 
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_SET;
 import static 
org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_TRANSITIVE_REUSE;
@@ -91,10 +92,16 @@ public class DictionaryRuleTest extends 
LocalFileMetadataTestCase {
 DictionaryDesc.create("lstg_site_id", "SELLER_ID", null),
 DictionaryDesc.create("price", "lstg_site_id", null));
 }
-
+
+@Test
+public void testBadDesc5() throws IOException {
+

[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2

2017-02-09 Thread lidong

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d149df1e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d149df1e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d149df1e

Branch: refs/heads/yang22-hbase102
Commit: d149df1ea0998ca71dfdbb6f1e87f9d703a0fcd4
Parents: 2f12bb4
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 17:37:29 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d149df1e/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d149df1e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/d149df1e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-02-09 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f12bb4c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f12bb4c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f12bb4c

Branch: refs/heads/yang22-hbase102
Commit: 2f12bb4c6712f41231cfa3f1174647e56c02950c
Parents: 5ef2480
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 17:34:11 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2017-02-09 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 a649e87cb -> d149df1ea (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2017-02-09 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 123849433 -> 2a0fd52e5 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-02-09 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f12bb4c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f12bb4c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f12bb4c

Branch: refs/heads/yang22-cdh5.7
Commit: 2f12bb4c6712f41231cfa3f1174647e56c02950c
Parents: 5ef2480
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 17:34:11 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-02-09 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f12bb4c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f12bb4c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f12bb4c

Branch: refs/heads/yang22-hbase1.x
Commit: 2f12bb4c6712f41231cfa3f1174647e56c02950c
Parents: 5ef2480
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 17:34:11 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2017-02-09 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x ba6e17883 -> 2f12bb4c6 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2f12bb4c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();

kylin git commit: minor,filter inDimNotInMea add return

2017-02-09 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master 1e9f0908b -> abf533225


minor,filter inDimNotInMea add return

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/abf53322
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/abf53322
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/abf53322

Branch: refs/heads/master
Commit: abf53322527148eaa49020bc2a327a210308e62d
Parents: 1e9f090
Author: luguosheng <550175...@qq.com>
Authored: Thu Feb 9 16:14:56 2017 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 16:17:49 2017 +0800

--
 webapp/app/js/filters/filter.js | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/abf53322/webapp/app/js/filters/filter.js
--
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index d097075..01c060a 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -202,6 +202,7 @@ KylinApp
   });
 }
   });
+  return out;
 }
   }).filter('assignedMeasureNames', function ($filter) {
 //return the measures that haven't assign to column family

kylin git commit: KYLIN-2437 collect number of bytes scanned to query metrics

kylin git commit: KYLIN-2428 try fix guava in Spark IT again

kylin git commit: KYLIN-2428 fix spark dep in IT

kylin git commit: KYLIN-2441 protocol for REST API result format

[4/9] kylin git commit: KYLIN-2440 Make PK/FK always exposed

[1/9] kylin git commit: KYLIN-2308 bug fix and improve user experience [Forced Update!]

[7/9] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

[3/9] kylin git commit: minor, decorateScanner can skip filter only

[6/9] kylin git commit: fix some testcases which can not be run concurrently

[8/9] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

kylin git commit: KYLIN-2428 fix shade

kylin git commit: KYLIN-2428 shade third party libraries in job/coprocessor/tool

kylin git commit: KYLIN-2316 Global dict couldn't be used for dimension column

[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

kylin git commit: minor,filter inDimNotInMea add return

21 matches

Site Navigation

Mail list logo

Footer information