kylin git commit: KYLIN-2308 bug fix and improve user experience

[kangkaisen]

Repository: kylin
Updated Branches:
  refs/heads/master 7742b31d8 -> 1e9f0908b


KYLIN-2308 bug fix and improve user experience


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e9f0908
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e9f0908
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e9f0908

Branch: refs/heads/master
Commit: 1e9f0908b9942113bbd251dc5eeba71973cc77e1
Parents: 7742b31
Author: kangkaisen 
Authored: Thu Feb 9 12:06:56 2017 +0800
Committer: kangkaisen 
Committed: Thu Feb 9 15:54:01 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 36 ++--
 webapp/app/js/controllers/cubeSchema.js |  8 +
 .../cubeDesigner/advanced_settings.html |  2 +-
 3 files changed, 43 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 39d36b0..7cdb1cd 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -342,12 +342,44 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 return assignedMeasures;
   };
 
-  if ($scope.getAllMeasureNames().length != 
$scope.getAssignedMeasureNames().length) {
+  $scope.rmDeprecatedMeasureNames = function () {
+var allMeasureNames = $scope.getAllMeasureNames();
+var tmpColumnFamily = $scope.cubeMetaFrame.hbase_mapping.column_family;
+
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily,index1) {
+  angular.forEach(colFamily.columns[0].measure_refs, function 
(measureName, index2) {
+var allIndex = allMeasureNames.indexOf(measureName);
+if (allIndex == -1) {
+  tmpColumnFamily[index1].columns[0].measure_refs.splice(index2, 1);
+}
+
+if (tmpColumnFamily[index1].columns[0].measure_refs == 0) {
+  tmpColumnFamily.splice(index1, 1);
+}
+  });
+});
+
+$scope.cubeMetaFrame.hbase_mapping.column_family = tmpColumnFamily;
+  };
+
+  if ($scope.getAssignedMeasureNames().length == 0) {
 $scope.initColumnFamily();
+  } else {
+$scope.rmDeprecatedMeasureNames();
+if ($scope.getAllMeasureNames().length > 
$scope.getAssignedMeasureNames().length) {
+  $scope.initColumnFamily();
+}
   }
 
-
   $scope.addColumnFamily = function () {
+var isCFEmpty = _.some($scope.cubeMetaFrame.hbase_mapping.column_family, 
function(colFamily) {
+  return colFamily.columns[0].measure_refs.length == 0;
+});
+
+if (isCFEmpty === true) {
+  return;
+}
+
 var colFamily = 
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 
1);
 $scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
   };

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/js/controllers/cubeSchema.js
--
diff --git a/webapp/app/js/controllers/cubeSchema.js 
b/webapp/app/js/controllers/cubeSchema.js
index 17371f2..4beeebe 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -287,6 +287,14 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, 
QueryService, UserServic
   errors.push("All measures need to be assigned to column family");
 }
 
+var isCFEmpty = 
_.some($scope.cubeMetaFrame.hbase_mapping.column_family, function(colFamily) {
+  return colFamily.columns[0].measure_refs.length == 0;
+});
+
+if (isCFEmpty == true) {
+  errors.push("Each column family can't not be empty");
+}
+
 
 var errorInfo = "";
 angular.forEach(errors,function(item){

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/partials/cubeDesigner/advanced_settings.html
--
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html 
b/webapp/app/partials/cubeDesigner/advanced_settings.html
index bf95256..f26722b 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -453,7 +453,7 @@
 
  

-   
+   
  

  

[1/4] kylin git commit: minor, use envelope for rest API reponse [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master-hbase0.98 4e41c3637 -> 39c8ac30e (forced update)


minor, use envelope for rest API reponse


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7742b31d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7742b31d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7742b31d

Branch: refs/heads/master-hbase0.98
Commit: 7742b31d85e4256556c52d3bf0b485ef20151508
Parents: 722efb8
Author: Hongbin Ma 
Authored: Thu Feb 9 10:47:21 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Feb 9 10:47:21 2017 +0800

--
 .../rest/controller/EncodingController.java | 11 ++--
 .../kylin/rest/response/EnvelopeResponse.java   | 58 
 2 files changed, 62 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7742b31d/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
index 2f532e2..b95394c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
@@ -23,6 +23,8 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.service.EncodingService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -34,8 +36,6 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
-import org.apache.kylin.rest.service.EncodingService;
-
 @Controller
 @RequestMapping(value = "/encodings")
 public class EncodingController extends BasicController {
@@ -52,7 +52,7 @@ public class EncodingController extends BasicController {
  */
 @RequestMapping(value = "valid_encodings", method = { RequestMethod.GET })
 @ResponseBody
-public Map getValidEncodings() {
+public EnvelopeResponse getValidEncodings() {
 
 Set allDatatypes = Sets.newHashSet();
 allDatatypes.addAll(DataType.DATETIME_FAMILY);
@@ -65,9 +65,6 @@ public class EncodingController extends BasicController {
 datatypeValidEncodings.put(dataTypeStr, 
encodingService.getValidEncodings(DataType.getType(dataTypeStr)));
 }
 
-Map ret = Maps.newHashMap();
-ret.put("code", "000");
-ret.put("data", datatypeValidEncodings);
-return ret;
+return new EnvelopeResponse(EnvelopeResponse.CODE_SUCCESS, 
datatypeValidEncodings, "");
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7742b31d/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
 
b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
new file mode 100644
index 000..564db70
--- /dev/null
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.response;
+
+public class EnvelopeResponse {
+
+public final static String CODE_SUCCESS = "000";
+
+private String code;
+private Object data;
+private String msg;
+
+public EnvelopeResponse(String code, Object data, String msg) {
+this.code = code;
+this.data = data;
+this.msg = msg;
+}
+
+public String getCode() {
+return code;
+}
+
+public void 

[3/4] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/39c8ac30/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 74ab017..a44de4f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -30,15 +30,14 @@ import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +68,7 @@ public class HBaseResourceStore extends ResourceStore {
 final String tableNameBase;
 final String hbaseUrl;
 
-Connection getConnection() throws IOException {
+HConnection getConnection() throws IOException {
 return HBaseConnection.get(hbaseUrl);
 }
 
@@ -121,7 +120,7 @@ public class HBaseResourceStore extends ResourceStore {
 byte[] endRow = Bytes.toBytes(lookForPrefix);
 endRow[endRow.length - 1]++;
 
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 Scan scan = new Scan(startRow, endRow);
 if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
 scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,12 +237,13 @@ public class HBaseResourceStore extends ResourceStore {
 IOUtils.copy(content, bout);
 bout.close();
 
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 byte[] row = Bytes.toBytes(resPath);
 Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
 
 table.put(put);
+table.flushCommits();
 } finally {
 IOUtils.closeQuietly(table);
 }
@@ -251,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected long checkAndPutResourceImpl(String resPath, byte[] content, 
long oldTS, long newTS) throws IOException, IllegalStateException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 byte[] row = Bytes.toBytes(resPath);
 byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -264,6 +264,8 @@ public class HBaseResourceStore extends ResourceStore {
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);
 }
 
+table.flushCommits();
+
 return newTS;
 } finally {
 IOUtils.closeQuietly(table);
@@ -272,7 +274,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected void deleteResourceImpl(String resPath) throws IOException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 boolean hdfsResourceExist = false;
 Result result = internalGetFromHTable(table, resPath, true, false);
@@ -285,6 +287,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 Delete del = new Delete(Bytes.toBytes(resPath));
 table.delete(del);
+table.flushCommits();
 
 if (hdfsResourceExist) { // remove hdfs cell value
 Path redirectPath = bigCellHDFSPath(resPath);
@@ -305,7 +308,7 @@ public class HBaseResourceStore extends ResourceStore {
 }
 
 private Result getFromHTable(String path, boolean fetchContent, boolean 
fetchTimestamp) throws IOException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 

[4/4] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

[lidong]

KYLIN-2307 Create a branch for master with HBase 0.98 API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39c8ac30
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39c8ac30
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39c8ac30

Branch: refs/heads/master-hbase0.98
Commit: 39c8ac30ef3303684333bb16d0c9c1be1dd150aa
Parents: 7742b31
Author: lidongsjtu 
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: root 
Committed: Thu Feb 9 04:56:32 2017 +

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |  25 
 dev-support/test_all_against_hdp_2_4_0_0_169.sh |  25 
 .../sandbox/capacity-scheduler.xml  |  17 ++-
 examples/test_case_data/sandbox/core-site.xml   |  28 +---
 examples/test_case_data/sandbox/hbase-site.xml  | 119 +
 examples/test_case_data/sandbox/hdfs-site.xml   |  84 +---
 examples/test_case_data/sandbox/hive-site.xml   |  89 +
 examples/test_case_data/sandbox/mapred-site.xml |  57 +++--
 examples/test_case_data/sandbox/yarn-site.xml   | 127 +++
 .../kylin/provision/BuildCubeWithEngine.java|  17 +--
 pom.xml | 122 +-
 .../kylin/rest/security/AclHBaseStorage.java|   4 +-
 .../rest/security/MockAclHBaseStorage.java  |   8 +-
 .../apache/kylin/rest/security/MockHTable.java  |  95 +++---
 .../rest/security/RealAclHBaseStorage.java  |   9 +-
 .../apache/kylin/rest/service/AclService.java   |  25 ++--
 .../apache/kylin/rest/service/CubeService.java  |  35 +++--
 .../apache/kylin/rest/service/QueryService.java |  24 ++--
 .../apache/kylin/rest/service/UserService.java  |  17 +--
 .../kylin/storage/hbase/HBaseConnection.java|  44 +++
 .../kylin/storage/hbase/HBaseResourceStore.java |  31 +++--
 .../storage/hbase/cube/SimpleHBaseStore.java|  20 +--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  13 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |   4 +-
 .../storage/hbase/steps/CubeHTableUtil.java |  16 +--
 .../storage/hbase/steps/DeprecatedGCStep.java   |  24 ++--
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  |  23 ++--
 .../storage/hbase/util/CleanHtableCLI.java  |  12 +-
 .../storage/hbase/util/CubeMigrationCLI.java|  37 +++---
 .../hbase/util/CubeMigrationCheckCLI.java   |  17 +--
 .../hbase/util/DeployCoprocessorCLI.java|  27 ++--
 .../hbase/util/ExtendCubeToHybridCLI.java   |   8 +-
 .../hbase/util/GridTableHBaseBenchmark.java |  34 ++---
 .../kylin/storage/hbase/util/HBaseClean.java|  18 ++-
 .../hbase/util/HBaseRegionSizeCalculator.java   |  35 +++--
 .../kylin/storage/hbase/util/HBaseUsage.java|   9 +-
 .../storage/hbase/util/HbaseStreamingInput.java |  30 ++---
 .../hbase/util/HtableAlterMetadataCLI.java  |   9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java |  19 +--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  |  15 +--
 .../kylin/storage/hbase/util/RowCounterCLI.java |  11 +-
 .../storage/hbase/util/StorageCleanupJob.java   |  20 ++-
 .../storage/hbase/util/UpdateHTableHostCLI.java |  17 +--
 tool/pom.xml|  10 --
 .../org/apache/kylin/tool/CubeMigrationCLI.java |  19 +--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |   8 +-
 48 files changed, 596 insertions(+), 877 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/39c8ac30/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean 

[2/4] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/39c8ac30/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
--
diff --git 
a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.ResourceStore;
@@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI {
 Serializer projectSerializer = new 
JsonSerializer(ProjectInstance.class);
 ProjectInstance project = store.getResource(projectResPath, 
ProjectInstance.class, projectSerializer);
 String projUUID = project.getUuid();
-Table aclHtable = null;
+HTableInterface aclHtable = null;
 try {
-aclHtable = 
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix()
 + "_acl"));
+aclHtable = 
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix()
 + "_acl");
 
 // cube acl
 Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI {
 aclHtable.put(put);
 }
 }
+aclHtable.flushCommits();
 } finally {
 IOUtils.closeQuietly(aclHtable);
 }

[5/5] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[lidong]

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a649e87c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a649e87c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a649e87c

Branch: refs/heads/yang22-hbase102
Commit: a649e87cb9f7774f4219ef02b5f82e09adaf9331
Parents: ba6e178
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 12:41:29 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a649e87c/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a649e87c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/a649e87c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of 

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/ba6e1788/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ba6e1788/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba6e1788
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba6e1788
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba6e1788

Branch: refs/heads/yang22-hbase102
Commit: ba6e17883862b44667ee6c1c0a0e0d80efe0754f
Parents: 5ef2480
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 12:32:13 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ba6e1788/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/5] kylin git commit: KYLIN-2410 refactor HadoopUtil to common module and let global dictionary use it [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 aa5a36133 -> a649e87cb (forced update)


KYLIN-2410 refactor HadoopUtil to common module and let global dictionary use it


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/911a8a4d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/911a8a4d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/911a8a4d

Branch: refs/heads/yang22-hbase102
Commit: 911a8a4db14de103b350550a92e07e608ee3b3cf
Parents: 8bfef64
Author: Yang Li 
Authored: Thu Jan 19 08:24:42 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 19 08:24:42 2017 +0800

--
 core-common/pom.xml |   5 +
 .../apache/kylin/common/util/HadoopUtil.java| 146 ++
 .../apache/kylin/dict/AppendTrieDictionary.java |   9 +-
 .../kylin/dict/AppendTrieDictionaryChecker.java |  13 +-
 .../org/apache/kylin/dict/CachedTreeMap.java|   7 +-
 .../kylin/dict/AppendTrieDictionaryTest.java|   6 +-
 .../apache/kylin/dict/CachedTreeMapTest.java|  29 ++--
 .../apache/kylin/engine/mr/DFSFileTable.java|   1 +
 .../kylin/engine/mr/DFSFileTableReader.java |   1 +
 .../org/apache/kylin/engine/mr/HadoopUtil.java  | 150 ---
 .../org/apache/kylin/engine/mr/KylinMapper.java |   1 +
 .../apache/kylin/engine/mr/KylinReducer.java|   1 +
 .../kylin/engine/mr/SortedColumnDFSFile.java|   1 +
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |   2 +-
 .../engine/mr/common/MapReduceExecutable.java   |   2 +-
 .../engine/mr/steps/CreateDictionaryJob.java|   2 +-
 .../mr/steps/FactDistinctColumnsReducer.java|   2 +-
 .../kylin/engine/mr/steps/InMemCuboidJob.java   |   2 +-
 .../engine/mr/steps/MergeStatisticsStep.java|   2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  |   2 +-
 .../engine/mr/steps/SaveStatisticsStep.java |   2 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   2 +-
 .../steps/FactDistinctColumnsReducerTest.java   |   2 +-
 .../HiveToBaseCuboidMapperPerformanceTest.java  |   2 +-
 .../engine/mr/steps/MergeCuboidJobTest.java |   2 +-
 .../kylin/engine/mr/steps/NDCuboidJobTest.java  |   2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |   2 +-
 .../kylin/provision/BuildCubeWithEngine.java|   2 +-
 .../apache/kylin/query/ITMassInQueryTest.java   |   2 +-
 .../kylin/storage/hbase/ITHdfsOpsTest.java  |   2 +-
 .../rest/controller/StreamingController.java|   2 +-
 .../kylin/rest/controller/TableController.java  |   2 +-
 .../apache/kylin/rest/service/CubeService.java  |   2 +-
 .../apache/kylin/rest/service/ModelService.java |   2 +-
 .../apache/kylin/source/hive/HiveMRInput.java   |   2 +-
 .../source/hive/HiveSourceTableLoader.java  |   2 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |   2 +-
 .../kylin/storage/hbase/HBaseConnection.java|   2 +-
 .../kylin/storage/hbase/HBaseResourceStore.java |   2 +-
 .../storage/hbase/steps/DeprecatedGCStep.java   |   2 +-
 .../steps/HDFSPathGarbageCollectionStep.java|   2 +-
 .../hbase/steps/SequenceFileCuboidWriter.java   |   2 +-
 .../storage/hbase/util/CubeMigrationCLI.java|   2 +-
 .../storage/hbase/util/StorageCleanupJob.java   |   2 +-
 .../storage/hbase/steps/CreateHTableTest.java   |   2 +-
 .../hbase/steps/CubeHFileMapper2Test.java   |   2 +-
 .../steps/RangeKeyDistributionJobTest.java  |   2 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java |   2 +-
 .../apache/kylin/tool/MrJobInfoExtractor.java   |   2 +-
 .../apache/kylin/tool/StorageCleanupJob.java|   2 +-
 .../org/apache/kylin/tool/util/ToolUtil.java|   2 +-
 52 files changed, 231 insertions(+), 217 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/911a8a4d/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 1bc5836..7b0eb98 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -81,6 +81,11 @@
 
 
 
+org.apache.hadoop
+hadoop-common
+provided
+
+
 junit
 junit
 test

http://git-wip-us.apache.org/repos/asf/kylin/blob/911a8a4d/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
new file mode 100644
index 000..a8477a9
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software 

[2/5] kylin git commit: KYLIN-2410 remove force qualified path in HadoopUtil.getFileSystem

[lidong]

KYLIN-2410 remove force qualified path in HadoopUtil.getFileSystem


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5ef2480c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5ef2480c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5ef2480c

Branch: refs/heads/yang22-cdh5.7
Commit: 5ef2480c088649a76596f49ac2420d606152b4c3
Parents: 911a8a4
Author: Yang Li 
Authored: Thu Jan 19 10:35:50 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 19 10:35:50 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/util/HadoopUtil.java| 3 ---
 1 file changed, 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5ef2480c/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index a8477a9..de87d61 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -82,9 +82,6 @@ public class HadoopUtil {
 }
 
 public static FileSystem getFileSystem(Path path, Configuration conf) {
-if (StringUtils.isBlank(path.toUri().getScheme()))
-throw new IllegalArgumentException("Path must be qualified: " + 
path);
-
 try {
 return path.getFileSystem(conf);
 } catch (IOException e) {

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/ba6e1788/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/ba6e1788/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba6e1788
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba6e1788
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba6e1788

Branch: refs/heads/yang22-cdh5.7
Commit: ba6e17883862b44667ee6c1c0a0e0d80efe0754f
Parents: 5ef2480
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 12:32:13 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ba6e1788/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[1/5] kylin git commit: KYLIN-2410 refactor HadoopUtil to common module and let global dictionary use it [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 494a00195 -> 123849433 (forced update)


KYLIN-2410 refactor HadoopUtil to common module and let global dictionary use it


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/911a8a4d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/911a8a4d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/911a8a4d

Branch: refs/heads/yang22-cdh5.7
Commit: 911a8a4db14de103b350550a92e07e608ee3b3cf
Parents: 8bfef64
Author: Yang Li 
Authored: Thu Jan 19 08:24:42 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 19 08:24:42 2017 +0800

--
 core-common/pom.xml |   5 +
 .../apache/kylin/common/util/HadoopUtil.java| 146 ++
 .../apache/kylin/dict/AppendTrieDictionary.java |   9 +-
 .../kylin/dict/AppendTrieDictionaryChecker.java |  13 +-
 .../org/apache/kylin/dict/CachedTreeMap.java|   7 +-
 .../kylin/dict/AppendTrieDictionaryTest.java|   6 +-
 .../apache/kylin/dict/CachedTreeMapTest.java|  29 ++--
 .../apache/kylin/engine/mr/DFSFileTable.java|   1 +
 .../kylin/engine/mr/DFSFileTableReader.java |   1 +
 .../org/apache/kylin/engine/mr/HadoopUtil.java  | 150 ---
 .../org/apache/kylin/engine/mr/KylinMapper.java |   1 +
 .../apache/kylin/engine/mr/KylinReducer.java|   1 +
 .../kylin/engine/mr/SortedColumnDFSFile.java|   1 +
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |   2 +-
 .../engine/mr/common/MapReduceExecutable.java   |   2 +-
 .../engine/mr/steps/CreateDictionaryJob.java|   2 +-
 .../mr/steps/FactDistinctColumnsReducer.java|   2 +-
 .../kylin/engine/mr/steps/InMemCuboidJob.java   |   2 +-
 .../engine/mr/steps/MergeStatisticsStep.java|   2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  |   2 +-
 .../engine/mr/steps/SaveStatisticsStep.java |   2 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   2 +-
 .../steps/FactDistinctColumnsReducerTest.java   |   2 +-
 .../HiveToBaseCuboidMapperPerformanceTest.java  |   2 +-
 .../engine/mr/steps/MergeCuboidJobTest.java |   2 +-
 .../kylin/engine/mr/steps/NDCuboidJobTest.java  |   2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |   2 +-
 .../kylin/provision/BuildCubeWithEngine.java|   2 +-
 .../apache/kylin/query/ITMassInQueryTest.java   |   2 +-
 .../kylin/storage/hbase/ITHdfsOpsTest.java  |   2 +-
 .../rest/controller/StreamingController.java|   2 +-
 .../kylin/rest/controller/TableController.java  |   2 +-
 .../apache/kylin/rest/service/CubeService.java  |   2 +-
 .../apache/kylin/rest/service/ModelService.java |   2 +-
 .../apache/kylin/source/hive/HiveMRInput.java   |   2 +-
 .../source/hive/HiveSourceTableLoader.java  |   2 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |   2 +-
 .../kylin/storage/hbase/HBaseConnection.java|   2 +-
 .../kylin/storage/hbase/HBaseResourceStore.java |   2 +-
 .../storage/hbase/steps/DeprecatedGCStep.java   |   2 +-
 .../steps/HDFSPathGarbageCollectionStep.java|   2 +-
 .../hbase/steps/SequenceFileCuboidWriter.java   |   2 +-
 .../storage/hbase/util/CubeMigrationCLI.java|   2 +-
 .../storage/hbase/util/StorageCleanupJob.java   |   2 +-
 .../storage/hbase/steps/CreateHTableTest.java   |   2 +-
 .../hbase/steps/CubeHFileMapper2Test.java   |   2 +-
 .../steps/RangeKeyDistributionJobTest.java  |   2 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java |   2 +-
 .../apache/kylin/tool/MrJobInfoExtractor.java   |   2 +-
 .../apache/kylin/tool/StorageCleanupJob.java|   2 +-
 .../org/apache/kylin/tool/util/ToolUtil.java|   2 +-
 52 files changed, 231 insertions(+), 217 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/911a8a4d/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 1bc5836..7b0eb98 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -81,6 +81,11 @@
 
 
 
+org.apache.hadoop
+hadoop-common
+provided
+
+
 junit
 junit
 test

http://git-wip-us.apache.org/repos/asf/kylin/blob/911a8a4d/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
new file mode 100644
index 000..a8477a9
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software 

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba6e1788
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba6e1788
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba6e1788

Branch: refs/heads/yang22-hbase1.x
Commit: ba6e17883862b44667ee6c1c0a0e0d80efe0754f
Parents: 5ef2480
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 12:32:13 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ba6e1788/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/4] kylin git commit: KYLIN-2410 refactor HadoopUtil to common module and let global dictionary use it [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x a57b00418 -> ba6e17883 (forced update)


KYLIN-2410 refactor HadoopUtil to common module and let global dictionary use it


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/911a8a4d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/911a8a4d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/911a8a4d

Branch: refs/heads/yang22-hbase1.x
Commit: 911a8a4db14de103b350550a92e07e608ee3b3cf
Parents: 8bfef64
Author: Yang Li 
Authored: Thu Jan 19 08:24:42 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 19 08:24:42 2017 +0800

--
 core-common/pom.xml |   5 +
 .../apache/kylin/common/util/HadoopUtil.java| 146 ++
 .../apache/kylin/dict/AppendTrieDictionary.java |   9 +-
 .../kylin/dict/AppendTrieDictionaryChecker.java |  13 +-
 .../org/apache/kylin/dict/CachedTreeMap.java|   7 +-
 .../kylin/dict/AppendTrieDictionaryTest.java|   6 +-
 .../apache/kylin/dict/CachedTreeMapTest.java|  29 ++--
 .../apache/kylin/engine/mr/DFSFileTable.java|   1 +
 .../kylin/engine/mr/DFSFileTableReader.java |   1 +
 .../org/apache/kylin/engine/mr/HadoopUtil.java  | 150 ---
 .../org/apache/kylin/engine/mr/KylinMapper.java |   1 +
 .../apache/kylin/engine/mr/KylinReducer.java|   1 +
 .../kylin/engine/mr/SortedColumnDFSFile.java|   1 +
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |   2 +-
 .../engine/mr/common/MapReduceExecutable.java   |   2 +-
 .../engine/mr/steps/CreateDictionaryJob.java|   2 +-
 .../mr/steps/FactDistinctColumnsReducer.java|   2 +-
 .../kylin/engine/mr/steps/InMemCuboidJob.java   |   2 +-
 .../engine/mr/steps/MergeStatisticsStep.java|   2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  |   2 +-
 .../engine/mr/steps/SaveStatisticsStep.java |   2 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   2 +-
 .../steps/FactDistinctColumnsReducerTest.java   |   2 +-
 .../HiveToBaseCuboidMapperPerformanceTest.java  |   2 +-
 .../engine/mr/steps/MergeCuboidJobTest.java |   2 +-
 .../kylin/engine/mr/steps/NDCuboidJobTest.java  |   2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |   2 +-
 .../kylin/provision/BuildCubeWithEngine.java|   2 +-
 .../apache/kylin/query/ITMassInQueryTest.java   |   2 +-
 .../kylin/storage/hbase/ITHdfsOpsTest.java  |   2 +-
 .../rest/controller/StreamingController.java|   2 +-
 .../kylin/rest/controller/TableController.java  |   2 +-
 .../apache/kylin/rest/service/CubeService.java  |   2 +-
 .../apache/kylin/rest/service/ModelService.java |   2 +-
 .../apache/kylin/source/hive/HiveMRInput.java   |   2 +-
 .../source/hive/HiveSourceTableLoader.java  |   2 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |   2 +-
 .../kylin/storage/hbase/HBaseConnection.java|   2 +-
 .../kylin/storage/hbase/HBaseResourceStore.java |   2 +-
 .../storage/hbase/steps/DeprecatedGCStep.java   |   2 +-
 .../steps/HDFSPathGarbageCollectionStep.java|   2 +-
 .../hbase/steps/SequenceFileCuboidWriter.java   |   2 +-
 .../storage/hbase/util/CubeMigrationCLI.java|   2 +-
 .../storage/hbase/util/StorageCleanupJob.java   |   2 +-
 .../storage/hbase/steps/CreateHTableTest.java   |   2 +-
 .../hbase/steps/CubeHFileMapper2Test.java   |   2 +-
 .../steps/RangeKeyDistributionJobTest.java  |   2 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java |   2 +-
 .../apache/kylin/tool/MrJobInfoExtractor.java   |   2 +-
 .../apache/kylin/tool/StorageCleanupJob.java|   2 +-
 .../org/apache/kylin/tool/util/ToolUtil.java|   2 +-
 52 files changed, 231 insertions(+), 217 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/911a8a4d/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 1bc5836..7b0eb98 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -81,6 +81,11 @@
 
 
 
+org.apache.hadoop
+hadoop-common
+provided
+
+
 junit
 junit
 test

http://git-wip-us.apache.org/repos/asf/kylin/blob/911a8a4d/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
new file mode 100644
index 000..a8477a9
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software 

kylin git commit: minor, use envelope for rest API reponse

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 722efb823 -> 7742b31d8


minor, use envelope for rest API reponse


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7742b31d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7742b31d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7742b31d

Branch: refs/heads/master
Commit: 7742b31d85e4256556c52d3bf0b485ef20151508
Parents: 722efb8
Author: Hongbin Ma 
Authored: Thu Feb 9 10:47:21 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Feb 9 10:47:21 2017 +0800

--
 .../rest/controller/EncodingController.java | 11 ++--
 .../kylin/rest/response/EnvelopeResponse.java   | 58 
 2 files changed, 62 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7742b31d/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
index 2f532e2..b95394c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
@@ -23,6 +23,8 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.service.EncodingService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -34,8 +36,6 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
-import org.apache.kylin.rest.service.EncodingService;
-
 @Controller
 @RequestMapping(value = "/encodings")
 public class EncodingController extends BasicController {
@@ -52,7 +52,7 @@ public class EncodingController extends BasicController {
  */
 @RequestMapping(value = "valid_encodings", method = { RequestMethod.GET })
 @ResponseBody
-public Map getValidEncodings() {
+public EnvelopeResponse getValidEncodings() {
 
 Set allDatatypes = Sets.newHashSet();
 allDatatypes.addAll(DataType.DATETIME_FAMILY);
@@ -65,9 +65,6 @@ public class EncodingController extends BasicController {
 datatypeValidEncodings.put(dataTypeStr, 
encodingService.getValidEncodings(DataType.getType(dataTypeStr)));
 }
 
-Map ret = Maps.newHashMap();
-ret.put("code", "000");
-ret.put("data", datatypeValidEncodings);
-return ret;
+return new EnvelopeResponse(EnvelopeResponse.CODE_SUCCESS, 
datatypeValidEncodings, "");
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7742b31d/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
 
b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
new file mode 100644
index 000..564db70
--- /dev/null
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.response;
+
+public class EnvelopeResponse {
+
+public final static String CODE_SUCCESS = "000";
+
+private String code;
+private Object data;
+private String msg;
+
+public EnvelopeResponse(String code, Object data, String msg) {
+this.code = code;
+this.data = data;
+this.msg = msg;
+}
+
+public String getCode() {
+return code;
+}
+
+public void setCode(String code) {
+this.code = 

[03/39] kylin git commit: KYLIN-2414 minor rename on properties

[lidong]

KYLIN-2414 minor rename on properties


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3fff6d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3fff6d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3fff6d1

Branch: refs/heads/master-hbase0.98
Commit: c3fff6d19d355e78461fa7f32d02feabc5bf63c8
Parents: e6a9382
Author: shaofengshi 
Authored: Thu Jan 26 10:53:19 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 10:53:19 2017 +0800

--
 build/conf/kylin.properties  |  8 +++-
 .../org/apache/kylin/common/KylinConfigBase.java |  4 ++--
 .../mr/steps/FactDistinctColumnsReducer.java | 19 +--
 3 files changed, 18 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 43ea17d..1232c47 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -127,9 +127,15 @@ kylin.engine.mr.max-reducer-number=500
 
 kylin.engine.mr.mapper-input-rows=100
 
+# Enable dictionary building in MR reducer
+kylin.engine.mr.build-dict-in-reducer=true
+
+# Number of reducers for fetching UHC column distinct values
+kylin.engine.mr.uhc-reducer-count=1
+
 ### CUBE | DICTIONARY ###
 
-# 'auto', 'inmem', 'layer' or 'random' for testing
+# 'auto', 'inmem' or 'layer'
 kylin.cube.algorithm=auto
 
 # A smaller threshold prefers layer, a larger threshold prefers in-mem

http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b25bcc0..6a88fc4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -740,8 +740,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
 }
 
-public boolean isReducerLocalBuildDict() {
-return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
+public boolean isBuildDictInReducerEnabled() {
+return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", 
"true"));
 }
 
 public String getYarnStatusCheckUrl() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index cf94b30..5b795c2 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -78,7 +78,7 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
 int[] uhcIndex = 
CubeManager.getInstance(config).getUHCIndex(cubeDesc);
 int colIndex = reducerIdToColumnIndex.get(taskId);
 if (uhcIndex[colIndex] == 1)
-isReducerLocalBuildDict = false; //for UHC columns, this 
feature should be disabled
+buildDictInReducer = false; //for UHC columns, this 
feature should be disabled
 }
-if (isReducerLocalBuildDict) {
+if (buildDictInReducer) {
 builder = 
DictionaryGenerator.newDictionaryBuilder(col.getType());
 builder.init(null, 0);
 }
-logger.info("Reducer " + taskId + " handling column " + col + ", 
isReducerLocalBuildDict=" + isReducerLocalBuildDict);
+logger.info("Reducer " + taskId + " handling column " + col + ", 
buildDictInReducer=" + buildDictInReducer);
 }
 }
 
@@ -192,7 +191,7 @@ public class FactDistinctColumnsReducer extends 
KylinReducer dict = builder.build();
 outputDict(col, dict);
 }

[24/39] kylin git commit: KYLIN-1875 Kylin support SnowFlake schema (alias name rule)

[lidong]

KYLIN-1875 Kylin support SnowFlake schema (alias name rule)

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8581f1d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8581f1d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8581f1d2

Branch: refs/heads/master-hbase0.98
Commit: 8581f1d203cee9fe3d834d6beb9f6bf7717f921a
Parents: 7a6dd1c
Author: luguosheng <550175...@qq.com>
Authored: Tue Feb 7 16:24:23 2017 +0800
Committer: zhongjian 
Committed: Tue Feb 7 17:08:24 2017 +0800

--
 webapp/app/partials/modelDesigner/data_model.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8581f1d2/webapp/app/partials/modelDesigner/data_model.html
--
diff --git a/webapp/app/partials/modelDesigner/data_model.html 
b/webapp/app/partials/modelDesigner/data_model.html
index 4d72266..167c088 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -157,8 +157,8 @@
 Alias
 
   
-   Alias is invalid.
+ ng-model="newLookup.alias"  
ng-pattern="/^[A-Z_\d]+$/">
+   Table alias is invalid(A Combination 
of numbers, uppercase letters or underscores).
 
 
 

[33/39] kylin git commit: minor, use local metadata to avoid conflict when running concurrently

[lidong]

minor, use local metadata to avoid conflict when running concurrently

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d135bdb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d135bdb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d135bdb0

Branch: refs/heads/master-hbase0.98
Commit: d135bdb0eed6118245a606fb0bca996420dee70f
Parents: 8263752
Author: etherge 
Authored: Wed Feb 8 18:40:07 2017 +0800
Committer: lidongsjtu 
Committed: Wed Feb 8 18:54:06 2017 +0800

--
 kylin-it/pom.xml  |  3 +++
 .../kylin/job/BaseTestDistributedScheduler.java   | 14 ++
 2 files changed, 13 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d135bdb0/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 91104ba..d58a895 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -36,6 +36,7 @@
 
 
 
+
 
 
 
@@ -316,6 +317,7 @@
 
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
+
-DbuildCubeUsingProvidedData=${buildCubeUsingProvidedData}
 
-DengineType=${engineType}
 
-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 -classpath
@@ -338,6 +340,7 @@
 
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
+
-DbuildCubeUsingProvidedData=${buildCubeUsingProvidedData}
 
-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 -classpath
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d135bdb0/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java 
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 1a0a39d..2f37a50 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
 
 import javax.annotation.Nullable;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
@@ -48,6 +49,7 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
+import com.google.common.io.Files;
 
 public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
 static ExecutableManager execMgr;
@@ -57,13 +59,13 @@ public class BaseTestDistributedScheduler extends 
HBaseMetadataTestCase {
 static KylinConfig kylinConfig1;
 static KylinConfig kylinConfig2;
 static CuratorFramework zkClient;
+static File localMetaDir;
 
 static final String SEGMENT_ID = "segmentId";
 static final String segmentId1 = "segmentId1";
 static final String segmentId2 = "segmentId2";
 static final String serverName1 = "serverName1";
 static final String serverName2 = "serverName2";
-static final String confSrcPath = 
"../examples/test_case_data/sandbox/kylin.properties";
 static final String confDstPath1 = 
"target/kylin_metadata_dist_lock_test1/kylin.properties";
 static final String confDstPath2 = 
"target/kylin_metadata_dist_lock_test2/kylin.properties";
 
@@ -77,14 +79,17 @@ public class BaseTestDistributedScheduler extends 
HBaseMetadataTestCase {
 new File(confDstPath1).getParentFile().mkdirs();
 new File(confDstPath2).getParentFile().mkdirs();
 KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
+
+localMetaDir = Files.createTempDir();
 String backup = srcConfig.getMetadataUrl();
-srcConfig.setProperty("kylin.metadata.url", 
"kylin_metadata_dist_lock_test@hbase");
+srcConfig.setProperty("kylin.metadata.url", 
localMetaDir.getAbsolutePath());
 srcConfig.writeProperties(new File(confDstPath1));
 

[14/39] kylin git commit: KYLIN-2422 NumberDictionary support for decimal with extra 0 after "."

[lidong]

KYLIN-2422 NumberDictionary support for decimal with extra 0 after "."


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24fa338e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24fa338e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24fa338e

Branch: refs/heads/master-hbase0.98
Commit: 24fa338e608030e3762ce5a17340fcf1d82029b1
Parents: 5da5393
Author: shaofengshi 
Authored: Sat Feb 4 14:16:11 2017 +0800
Committer: shaofengshi 
Committed: Sat Feb 4 19:37:59 2017 +0800

--
 .../org/apache/kylin/dict/NumberDictionary.java | 25 
 .../apache/kylin/dict/NumberDictionaryTest.java |  5 +++-
 2 files changed, 29 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/24fa338e/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index c55937d..de28440 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -53,6 +53,7 @@ public class NumberDictionary extends TrieDictionary {
 return;
 }
 
+
 if (len > buf.length) {
 throw new IllegalArgumentException("Too many digits for 
NumberDictionary: " + Bytes.toString(value, offset, len) + ". Internal buffer 
is only " + buf.length + " bytes");
 }
@@ -104,6 +105,30 @@ public class NumberDictionary extends TrieDictionary 
{
 
 bufOffset = start;
 bufLen = buf.length - start;
+
+// remove 0 in tail after the decimal point
+if (decimalPoint != end) {
+if (negative == true) {
+while (buf[bufOffset + bufLen - 2] == '9' && (bufOffset + 
bufLen - 2 > decimalPoint)) {
+bufLen--;
+}
+
+if (bufOffset + bufLen - 2 == decimalPoint) {
+bufLen--;
+}
+
+buf[bufOffset + bufLen - 1] = ';';
+} else {
+while (buf[bufOffset + bufLen - 1] == '0' && (bufOffset + 
bufLen - 1 > decimalPoint)) {
+bufLen--;
+}
+
+if (bufOffset + bufLen - 1 == decimalPoint) {
+bufLen--;
+}
+
+}
+}
 }
 
 int decodeNumber(byte[] returnValue, int offset) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/24fa338e/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java 
b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 1c04745..36eedf5 100644
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -91,11 +91,14 @@ public class NumberDictionaryTest extends 
LocalFileMetadataTestCase {
 checkCodec("-12345", "-9987654;");
 checkCodec("-12345.123", "-9987654.876;");
 checkCodec("0", "");
-checkCodec("0.0", ".0");
 //test resolved jira-1800
 checkCodec("-0.00454543543543543599877218", 
"-999.99545456456456456400122781;");
 checkCodec("-0.0099877218", 
"-999.9900122781;");
 checkCodec("12343434372493274.438403840384023840253554345345345345", 
"00012343434372493274.438403840384023840253554345345345345");
+assertEquals("0052.57", encodeNumber("52.5700"));
+assertEquals("", encodeNumber("0.00"));
+assertEquals("", encodeNumber("0.0"));
+assertEquals("-9987654.876;", 
encodeNumber("-12345.12300"));
 }
 
 private void checkCodec(String number, String code) {

[30/39] kylin git commit: KYLIN-2388 Hot load kylin config from web

[lidong]

KYLIN-2388 Hot load kylin config from web


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39afa519
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39afa519
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39afa519

Branch: refs/heads/master-hbase0.98
Commit: 39afa5197e197d5143843a3bb62c528749b03418
Parents: 7b860ad
Author: kangkaisen 
Authored: Thu Jan 12 14:35:59 2017 +0800
Committer: kangkaisen 
Committed: Tue Feb 7 19:57:29 2017 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  6 +-
 .../apache/kylin/common/KylinConfigTest.java| 28 -
 .../util/HotLoadKylinPropertiesTestCase.java| 64 
 .../kylin/cube/CubeSpecificConfigTest.java  | 37 ++-
 .../apache/kylin/job/JobEngineConfigTest.java   | 47 ++
 .../kylin/rest/controller/CacheController.java  |  9 ++-
 webapp/app/js/controllers/admin.js  | 26 
 webapp/app/js/services/cache.js |  3 +-
 webapp/app/partials/admin/admin.html|  3 +
 9 files changed, 191 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 0f40654..c7a18c6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -201,7 +201,7 @@ public class KylinConfig extends KylinConfigBase {
 return kylinHome + File.separator + "conf";
 }
 
-static File getKylinPropertiesFile() {
+public static File getKylinPropertiesFile() {
 String kylinConfHome = System.getProperty(KYLIN_CONF);
 if (!StringUtils.isEmpty(kylinConfHome)) {
 logger.info("Use KYLIN_CONF=" + kylinConfHome);
@@ -385,4 +385,8 @@ public class KylinConfig extends KylinConfigBase {
 //}
 //logger.info(buf.toString());
 }
+
+public synchronized void hotLoadKylinProperties() {
+reloadKylinConfig(getKylinProperties());
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java 
b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index a426fc6..4d5f130 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -26,24 +26,12 @@ import static org.junit.Assert.assertTrue;
 
 import java.util.Map;
 
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
 import org.junit.Test;
 
 import com.google.common.collect.Maps;
 
-public class KylinConfigTest extends LocalFileMetadataTestCase {
-@Before
-public void setUp() throws Exception {
-this.createTestMetadata();
-}
-
-@After
-public void after() throws Exception {
-this.cleanupTestMetadata();
-}
-
+public class KylinConfigTest extends HotLoadKylinPropertiesTestCase{
 @Test
 public void testMRConfigOverride() {
 KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -78,9 +66,19 @@ public class KylinConfigTest extends 
LocalFileMetadataTestCase {
 KylinConfig config = KylinConfig.getInstanceFromEnv();
 Map override = Maps.newHashMap();
 KylinConfig configExt = KylinConfigExt.createInstance(config, 
override);
-
 assertTrue(config.properties == configExt.properties);
 config.setProperty("1234", "1234");
 assertEquals("1234", configExt.getOptional("1234"));
 }
+
+@Test
+public void testPropertiesHotLoad() {
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+assertEquals("who...@kylin.apache.org", config.getKylinOwner());
+
+updateProperty("kylin.storage.hbase.owner-tag", 
"ky...@kylin.apache.org");
+KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+
+assertEquals("ky...@kylin.apache.org", config.getKylinOwner());
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
--

[34/39] kylin git commit: minor, clean up kylin.sh a little

[lidong]

minor, clean up kylin.sh a little


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df3ecd32
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df3ecd32
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df3ecd32

Branch: refs/heads/master-hbase0.98
Commit: df3ecd324a6442fee52f146d930cc5f84dafd945
Parents: d135bdb
Author: Yang Li 
Authored: Wed Feb 8 20:42:02 2017 +0800
Committer: Yang Li 
Committed: Wed Feb 8 20:42:02 2017 +0800

--
 build/bin/kylin.sh | 12 
 1 file changed, 4 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/df3ecd32/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7813b79..cc571e5 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -60,8 +60,7 @@ then
 PID=`cat $KYLIN_HOME/pid`
 if ps -p $PID > /dev/null
 then
-  echo "Kylin is running, stop it first"
-  exit 1
+  quit "Kylin is running, stop it first"
 fi
 fi
 
@@ -81,7 +80,7 @@ then
 spring_profile=`bash ${dir}/get-properties.sh kylin.security.profile`
 if [ -z "$spring_profile" ]
 then
-quit 'please set kylin.security.profile in kylin.properties, options 
are: testing, ldap, saml.'
+quit 'Please set kylin.security.profile in kylin.properties, options 
are: testing, ldap, saml.'
 else
 verbose "kylin.security.profile is set to $spring_profile"
 fi
@@ -95,10 +94,7 @@ then
 kylin_rest_address_arr=(${kylin_rest_address//;/ })
 nc -z -w 5 ${kylin_rest_address_arr[0]} ${kylin_rest_address_arr[1]} 
1>/dev/null 2>&1; nc_result=$?
 if [ $nc_result -eq 0 ]; then
-echo "port ${kylin_rest_address} is not available, could not start 
Kylin"
-exit 1
-else
-echo "port ${kylin_rest_address} is available"
+quit "Port ${kylin_rest_address} is not available, could not start 
Kylin."
 fi
 
 #debug if encounter NoClassDefError
@@ -155,7 +151,7 @@ then
 
 elif [ "$1" = "diag" ]
 then
-echo "kylin.sh diag no longer supported, use diag.sh instead"
+echo "'kylin.sh diag' no longer supported, use diag.sh instead"
 exit 0
 
 # tool command

[21/39] kylin git commit: minor, add filterPushdown setter in GTScanRequest

[lidong]

minor, add filterPushdown setter in GTScanRequest

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef0fc868
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef0fc868
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef0fc868

Branch: refs/heads/master-hbase0.98
Commit: ef0fc86876db488bf65353ec50e5f29411835b82
Parents: 4047e8d
Author: Roger Shi 
Authored: Mon Feb 6 15:40:56 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Feb 6 15:46:26 2017 +0800

--
 .../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java  | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef0fc868/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index dc90ed6..c45f90c 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -253,6 +253,10 @@ public class GTScanRequest {
 return filterPushDown;
 }
 
+public void setFilterPushDown(TupleFilter filter) {
+filterPushDown = filter;
+}
+
 public ImmutableBitSet getDimensions() {
 return this.getColumns().andNot(this.getAggrMetrics());
 }

[25/39] kylin git commit: KYLIN-2432 Couldn't select partition column in some old browser

[lidong]

KYLIN-2432 Couldn't select partition column in some old browser

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d11dd1d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d11dd1d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d11dd1d

Branch: refs/heads/master-hbase0.98
Commit: 6d11dd1d27a74fe50cd4fa58d2acf26f728f82e7
Parents: 8581f1d
Author: luguosheng <550175...@qq.com>
Authored: Tue Feb 7 16:54:05 2017 +0800
Committer: zhongjian 
Committed: Tue Feb 7 17:08:45 2017 +0800

--
 webapp/app/js/config.js | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6d11dd1d/webapp/app/js/config.js
--
diff --git a/webapp/app/js/config.js b/webapp/app/js/config.js
index 5f72eb2..b9ae6a5 100644
--- a/webapp/app/js/config.js
+++ b/webapp/app/js/config.js
@@ -35,7 +35,17 @@ var Config = {
   },
   contact_mail: ''
 };
-
+//resolve startsWith and endsWidth not work in low version chrome
+if (typeof String.prototype.startsWith != 'function') {
+  String.prototype.startsWith = function (prefix){
+return this.slice(0, prefix.length) === prefix;
+  };
+}
+if (typeof String.prototype.endsWith != 'function') {
+  String.prototype.endsWith = function(suffix) {
+return this.indexOf(suffix, this.length - suffix.length) !== -1;
+  };
+}
 // Angular module to load routes.
 KylinApp.config(function ($routeProvider, $httpProvider, $locationProvider, 
$logProvider) {
 //resolve http always use cache data in IE11,IE10

[23/39] kylin git commit: minor,update reload and unload table style

[lidong]

minor,update reload and unload table style

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a6dd1ca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a6dd1ca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a6dd1ca

Branch: refs/heads/master-hbase0.98
Commit: 7a6dd1ca3a35bb2a0223b7df0762f922c341d0af
Parents: 1f44468
Author: chenzhx <346839...@qq.com>
Authored: Mon Feb 6 18:19:49 2017 +0800
Committer: zhongjian 
Committed: Mon Feb 6 18:54:47 2017 +0800

--
 webapp/app/css/AdminLTE.css | 26 --
 webapp/app/js/controllers/sourceMeta.js | 96 +++-
 .../js/directives/kylin_abn_tree_directive.js   |  6 +-
 .../app/partials/tables/source_table_tree.html  |  4 -
 webapp/app/partials/tables/table_detail.html|  3 +-
 5 files changed, 57 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7a6dd1ca/webapp/app/css/AdminLTE.css
--
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index c7740d1..34e94e5 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -4801,29 +4801,3 @@ Gradient Background colors
   }
 }
 
-.abn-tree .abn-tree-row .tree-table-btn{
-width: 4%;
-float: right;
-border-radius: 5px;
-margin-right: 5px;
-padding: 3px 16px 2px 5px;
-color: #ff;
-position:static;
-}
-.abn-tree .abn-tree-row .tree-table-btn .tooltip{
-overflow:visible;
-}
-.abn-tree .abn-tree-row .btn-info:hover{
-   background-color:#269abc;
-}
-.abn-tree > .abn-tree-row.active > .btn-info{
-background-color:#269abc;
-border-left-color:#269abc;
-}
-.abn-tree .abn-tree-row .btn-success:hover{
-background-color:#008d4c;
-}
-.abn-tree > .abn-tree-row.active > .btn-success{
-background-color:#008d4c;
-border-left-color:#008d4c;
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/7a6dd1ca/webapp/app/js/controllers/sourceMeta.js
--
diff --git a/webapp/app/js/controllers/sourceMeta.js 
b/webapp/app/js/controllers/sourceMeta.js
index 5421673..e2fdd66 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -132,9 +132,10 @@ KylinApp
   });
 };
 
-$scope.reloadTable = function (tableName,projectName){
+$scope.reloadTable = function (tableName){
+  var delay = $q.defer();
   loadingRequest.show();
-  TableService.loadHiveTable({tableName: tableName, action: projectName}, 
{calculate: $scope.isCalculate}, function (result) {
+  TableService.loadHiveTable({tableName: tableName, action: 
$scope.projectModel.selectedProject}, {calculate: $scope.isCalculate}, function 
(result) {
 var loadTableInfo = "";
 angular.forEach(result['result.loaded'], function (table) {
   loadTableInfo += "\n" + table;
@@ -153,7 +154,7 @@ KylinApp
   SweetAlert.swal('Partial loaded!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo + "\n\n Failed to load following 
table(s):" + unloadedTableInfo, 'warning');
 }
 loadingRequest.hide();
-$scope.aceSrcTbLoaded(true);
+delay.resolve("");
   }, function (e) {
 if (e.data && e.data.exception) {
   var message = e.data.exception;
@@ -164,49 +165,58 @@ KylinApp
 }
 loadingRequest.hide();
   })
+   return delay.promise;
 }
 
 
 
-$scope.removeList = function (tableName,projectName) {
-  if (tableName.trim() === "") {
-SweetAlert.swal('', 'Please input table(s) you want to unload.', 
'info');
-return;
-  }
-  if (!projectName) {
-SweetAlert.swal('', 'Please choose your project first!.', 'info');
-return;
-  }
-  loadingRequest.show();
-  TableService.unLoadHiveTable({tableName: tableName, action: 
projectName}, {}, function (result) {
-var removedTableInfo = "";
-angular.forEach(result['result.unload.success'], function (table) {
-  removedTableInfo += "\n" + table;
-})
-var unRemovedTableInfo = "";
-angular.forEach(result['result.unload.fail'], function (table) {
-  unRemovedTableInfo += "\n" + table;
-})
-if (result['result.unload.fail'].length != 0 && 
result['result.unload.success'].length == 0) {
-  SweetAlert.swal('Failed!', 'Failed to unload following table(s): ' + 
unRemovedTableInfo, 'error');
-}
-if (result['result.unload.success'].length != 0 && 
result['result.unload.fail'].length == 0) {
-  SweetAlert.swal('Success!', 'The following table(s) have been 

[04/39] kylin git commit: minor, make DefaultSchedulerTest.testDiscard() more stable

[lidong]

minor, make DefaultSchedulerTest.testDiscard() more stable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0582512e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0582512e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0582512e

Branch: refs/heads/master-hbase0.98
Commit: 0582512ea06bcd1c662903ff69012ba1cc7cc33c
Parents: c3fff6d
Author: Li Yang 
Authored: Thu Jan 26 11:49:04 2017 +0800
Committer: Li Yang 
Committed: Thu Jan 26 11:49:56 2017 +0800

--
 .../apache/kylin/job/SelfStopExecutable.java| 31 
 .../impl/threadpool/DefaultSchedulerTest.java   |  5 ++--
 2 files changed, 27 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
index b4f6a98..9a3eb48 100644
--- a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
+++ b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
@@ -26,21 +26,40 @@ import org.apache.kylin.job.execution.ExecuteResult;
  */
 public class SelfStopExecutable extends BaseTestExecutable {
 
+volatile boolean doingWork;
+
 public SelfStopExecutable() {
 super();
 }
 
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
+doingWork = true;
 try {
-Thread.sleep(5000);
-} catch (InterruptedException e) {
-}
-if (isDiscarded()) {
-return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped");
-} else {
+for (int i = 0; i < 20; i++) {
+sleepOneSecond();
+
+if (isDiscarded())
+return new ExecuteResult(ExecuteResult.State.STOPPED, 
"stopped");
+}
+
 return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+} finally {
+doingWork = false;
+}
+}
+
+private void sleepOneSecond() {
+try {
+Thread.sleep(1000);
+} catch (InterruptedException e) {
+throw new RuntimeException(e);
 }
 }
 
+public void waitForDoWork() {
+while (doingWork) {
+sleepOneSecond();
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index bcd6a59..2416311 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -97,7 +97,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest {
 @Test
 public void testDiscard() throws Exception {
 DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SelfStopExecutable();
+SelfStopExecutable task1 = new SelfStopExecutable();
 job.addTask(task1);
 jobService.addJob(job);
 waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
@@ -105,8 +105,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest 
{
 waitForJobFinish(job.getId());
 Assert.assertEquals(ExecutableState.DISCARDED, 
jobService.getOutput(job.getId()).getState());
 Assert.assertEquals(ExecutableState.DISCARDED, 
jobService.getOutput(task1.getId()).getState());
-Thread.sleep(5000);
-System.out.println(job);
+task1.waitForDoWork();
 }
 
 @SuppressWarnings("rawtypes")

[15/39] kylin git commit: KYLIN-2421 Add spark engine to Integration Test

[lidong]

KYLIN-2421 Add spark engine to Integration Test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5da53936
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5da53936
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5da53936

Branch: refs/heads/master-hbase0.98
Commit: 5da53936502136c0d56236e148da2751aa1462c9
Parents: 855301d
Author: shaofengshi 
Authored: Fri Jan 20 11:28:57 2017 +0800
Committer: shaofengshi 
Committed: Sat Feb 4 19:37:59 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  10 ++
 .../measure/bitmap/RoaringBitmapCounter.java|   3 +-
 .../bitmap/RoaringBitmapCounterFactory.java |   3 +-
 .../measure/percentile/PercentileCounter.java   |  22 ++-
 .../percentile/PercentileSerializer.java|   6 +-
 .../kylin/measure/topn/TopNAggregator.java  |   5 +-
 .../percentile/PercentileCounterTest.java   |  47 ++
 .../kylin/engine/mr/BatchCubingJobBuilder2.java |   8 +-
 .../engine/spark/KylinKryoRegistrator.java  | 161 +++
 .../spark/SparkBatchCubingJobBuilder2.java  |  12 +-
 .../apache/kylin/engine/spark/SparkCubing.java  | 123 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |  65 
 .../localmeta/cube_desc/ci_inner_join_cube.json |  14 +-
 examples/test_case_data/sandbox/core-site.xml   |   2 +
 .../test_case_data/sandbox/kylin.properties |  29 ++--
 kylin-it/pom.xml|  21 +++
 .../kylin/provision/BuildCubeWithEngine.java|  25 +++
 17 files changed, 355 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5da53936/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6a88fc4..fe15b1e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -68,6 +68,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 return sparkHome;
 }
 
+sparkHome = System.getProperty("SPARK_HOME");
+if (StringUtils.isNotEmpty(sparkHome)) {
+logger.info("SPARK_HOME was set to " + sparkHome);
+return sparkHome;
+}
+
 return getKylinHome() + File.separator + "spark";
 }
 
@@ -760,6 +766,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.engine.spark.env.hadoop-conf-dir", "");
 }
 
+public void setHadoopConfDir(String hadoopConfDir) {
+setProperty("kylin.engine.spark.env.hadoop-conf-dir", hadoopConfDir);
+}
+
 public String getSparkAdditionalJars() {
 return getOptional("kylin.engine.spark.additional-jars", "");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5da53936/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index fb9dcfc..eec45f2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -24,6 +24,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.Serializable;
 import java.nio.BufferOverflowException;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
@@ -31,7 +32,7 @@ import java.util.Iterator;
 /**
  * A {@link BitmapCounter} based on roaring bitmap.
  */
-public class RoaringBitmapCounter implements BitmapCounter {
+public class RoaringBitmapCounter implements BitmapCounter, Serializable {
 
 private ImmutableRoaringBitmap bitmap;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5da53936/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
index a71df95..822afa2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
+++ 

[17/39] kylin git commit: minor fix ci

[lidong]

minor fix ci


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/663a6f93
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/663a6f93
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/663a6f93

Branch: refs/heads/master-hbase0.98
Commit: 663a6f939f2fed83f096daf8a3a55ed3f785e5a0
Parents: 5d83c80
Author: shaofengshi 
Authored: Sun Feb 5 09:57:34 2017 +0800
Committer: shaofengshi 
Committed: Sun Feb 5 09:57:34 2017 +0800

--
 .../org/apache/kylin/provision/BuildCubeWithStream.java | 12 +---
 .../test/java/org/apache/kylin/query/KylinTestBase.java |  2 +-
 2 files changed, 10 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/663a6f93/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 37ff8ae..53c89cf 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -87,6 +87,8 @@ public class BuildCubeWithStream {
 protected static boolean fastBuildMode = false;
 private boolean generateData = true;
 
+private static final int BUILD_ROUND = 5;
+
 public void before() throws Exception {
 deployEnv();
 
@@ -181,8 +183,12 @@ public class BuildCubeWithStream {
 ExecutorService executorService = Executors.newCachedThreadPool();
 
 List futures = Lists.newArrayList();
-for (int i = 0; i < 5; i++) {
-Thread.sleep(2 * 60 * 1000); // wait for new messages
+for (int i = 0; i < BUILD_ROUND; i++) {
+if (i == (BUILD_ROUND - 1)) {
+// stop generating message to kafka
+generateData = false;
+}
+Thread.sleep(1 * 60 * 1000); // wait for new messages
 FutureTask futureTask = new FutureTask(new 
Callable() {
 @Override
 public ExecutableState call() {
@@ -202,7 +208,7 @@ public class BuildCubeWithStream {
 futures.add(futureTask);
 }
 
-generateData = false; // stop generating message to kafka
+generateData = false;
 executorService.shutdown();
 int succeedBuild = 0;
 for (int i = 0; i < futures.size(); i++) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/663a6f93/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java 
b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 1cc5c76..6b9397d 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -646,7 +646,7 @@ public class KylinTestBase {
 cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + 
olapTmp.getAbsolutePath(), props);
 
 //setup h2
-h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + 
(h2InstanceCount++) + ";CACHE_SIZE=32072;DB_CLOSE_DELAY=-1", "sa", "");
+h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + 
(h2InstanceCount++) + ";CACHE_SIZE=32072", "sa", "");
 // Load H2 Tables (inner join)
 H2Database h2DB = new H2Database(h2Connection, config);
 h2DB.loadAllTables();

[10/39] kylin git commit: KYLIN-2420 Record zookeeper lock holder information

[lidong]

KYLIN-2420 Record zookeeper lock holder information

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56c7aa50
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56c7aa50
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56c7aa50

Branch: refs/heads/master-hbase0.98
Commit: 56c7aa505706f9aa878b4d3804cdf3f357a7931f
Parents: 7611338
Author: Roger Shi 
Authored: Sat Feb 4 13:43:03 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 13:46:59 2017 +0800

--
 .../storage/hbase/util/ZookeeperJobLock.java| 26 +++-
 1 file changed, 25 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56c7aa50/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
index bdd3981..7bf7498 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.storage.hbase.util;
 
+import java.lang.management.ManagementFactory;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
 
@@ -35,6 +37,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.job.lock.JobLock;
 import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,9 +66,14 @@ public class ZookeeperJobLock implements JobLock {
 }
 
 RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
-this.zkClient = CuratorFrameworkFactory.newClient(zkConnectString, 
retryPolicy);
+try {
+this.zkClient = 
CuratorFrameworkFactory.builder().connectString(zkConnectString).retryPolicy(retryPolicy).defaultData(getIpProcess()).build();
+} catch (UnknownHostException e) {
+throw new RuntimeException(e);
+}
 this.zkClient.start();
 this.sharedLock = new InterProcessMutex(zkClient, this.scheduleID);
+
 boolean hasLock = false;
 try {
 hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
@@ -74,9 +82,20 @@ public class ZookeeperJobLock implements JobLock {
 }
 if (!hasLock) {
 logger.warn("fail to acquire lock, scheduler has not been started; 
maybe another kylin process is still running?");
+try {
+for (String node : sharedLock.getParticipantNodes()) {
+logger.warn("lock holder info: {}", new 
String(zkClient.getData().forPath(node)));
+}
+} catch (Exception e) {
+logger.warn("error check participant", e);
+if (!(e instanceof KeeperException.NoNodeException)) {
+throw new RuntimeException(e);
+}
+}
 zkClient.close();
 return false;
 }
+
 return true;
 }
 
@@ -115,4 +134,9 @@ public class ZookeeperJobLock implements JobLock {
 private String schedulerId() {
 return ZOOKEEPER_LOCK_PATH + "/" + 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
 }
+
+private byte[] getIpProcess() throws UnknownHostException {
+logger.info("get IP and processId: {}", 
ManagementFactory.getRuntimeMXBean().getName().getBytes());
+return ManagementFactory.getRuntimeMXBean().getName().getBytes();
+}
 }

[02/39] kylin git commit: KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217

[lidong]

KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6a9382b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6a9382b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6a9382b

Branch: refs/heads/master-hbase0.98
Commit: e6a9382b8d357e5bb13be2f678ebabb8bad75dc7
Parents: 5eae37e
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 24 11:14:40 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 09:56:54 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 3 ---
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java  | 6 ++
 2 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b1acbbf..b25bcc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -741,9 +741,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public boolean isReducerLocalBuildDict() {
-if (getUHCReducerCount() != 1) {
-return false;
-}
 return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8c56bdf..cf94b30 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -132,6 +132,12 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
+int[] uhcIndex = 
CubeManager.getInstance(config).getUHCIndex(cubeDesc);
+int colIndex = reducerIdToColumnIndex.get(taskId);
+if (uhcIndex[colIndex] == 1)
+isReducerLocalBuildDict = false; //for UHC columns, this 
feature should be disabled
+}
 if (isReducerLocalBuildDict) {
 builder = 
DictionaryGenerator.newDictionaryBuilder(col.getType());
 builder.init(null, 0);

[38/39] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4e41c363/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 74ab017..a44de4f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -30,15 +30,14 @@ import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +68,7 @@ public class HBaseResourceStore extends ResourceStore {
 final String tableNameBase;
 final String hbaseUrl;
 
-Connection getConnection() throws IOException {
+HConnection getConnection() throws IOException {
 return HBaseConnection.get(hbaseUrl);
 }
 
@@ -121,7 +120,7 @@ public class HBaseResourceStore extends ResourceStore {
 byte[] endRow = Bytes.toBytes(lookForPrefix);
 endRow[endRow.length - 1]++;
 
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 Scan scan = new Scan(startRow, endRow);
 if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
 scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,12 +237,13 @@ public class HBaseResourceStore extends ResourceStore {
 IOUtils.copy(content, bout);
 bout.close();
 
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 byte[] row = Bytes.toBytes(resPath);
 Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
 
 table.put(put);
+table.flushCommits();
 } finally {
 IOUtils.closeQuietly(table);
 }
@@ -251,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected long checkAndPutResourceImpl(String resPath, byte[] content, 
long oldTS, long newTS) throws IOException, IllegalStateException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 byte[] row = Bytes.toBytes(resPath);
 byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -264,6 +264,8 @@ public class HBaseResourceStore extends ResourceStore {
 throw new IllegalStateException("Overwriting conflict " + 
resPath + ", expect old TS " + oldTS + ", but it is " + real);
 }
 
+table.flushCommits();
+
 return newTS;
 } finally {
 IOUtils.closeQuietly(table);
@@ -272,7 +274,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 @Override
 protected void deleteResourceImpl(String resPath) throws IOException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
 try {
 boolean hdfsResourceExist = false;
 Result result = internalGetFromHTable(table, resPath, true, false);
@@ -285,6 +287,7 @@ public class HBaseResourceStore extends ResourceStore {
 
 Delete del = new Delete(Bytes.toBytes(resPath));
 table.delete(del);
+table.flushCommits();
 
 if (hdfsResourceExist) { // remove hdfs cell value
 Path redirectPath = bigCellHDFSPath(resPath);
@@ -305,7 +308,7 @@ public class HBaseResourceStore extends ResourceStore {
 }
 
 private Result getFromHTable(String path, boolean fetchContent, boolean 
fetchTimestamp) throws IOException {
-Table table = 
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table = 

[31/39] kylin git commit: minor, import correct Lists

[lidong]

minor, import correct Lists


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0dcce15f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0dcce15f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0dcce15f

Branch: refs/heads/master-hbase0.98
Commit: 0dcce15f7fc01f6d8c388661025e5dab99ef05ad
Parents: 39afa51
Author: Hongbin Ma 
Authored: Wed Feb 8 15:18:10 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Feb 8 15:18:10 2017 +0800

--
 kylin-it/src/test/java/org/apache/kylin/query/H2Database.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0dcce15f/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java 
b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
index 8b60c49..78ed1b6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.clearspring.analytics.util.Lists;
+import com.google.common.collect.Lists;
 import org.apache.commons.io.IOUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.metadata.MetadataManager;

[13/39] kylin git commit: KYLIN-2421 fix unit test

[lidong]

KYLIN-2421 fix unit test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56a3e6c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56a3e6c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56a3e6c8

Branch: refs/heads/master-hbase0.98
Commit: 56a3e6c8d0c39271ea95d83bbe0f3f8c7db8b41b
Parents: 24fa338
Author: shaofengshi 
Authored: Sat Feb 4 14:43:36 2017 +0800
Committer: shaofengshi 
Committed: Sat Feb 4 19:37:59 2017 +0800

--
 core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56a3e6c8/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
--
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 86ea1df..20ee43e 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -48,6 +48,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -102,6 +103,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase 
{
 this.cleanupTestMetadata();
 }
 
+@Ignore ("To enable spark in IT, the inner cube removed the percentile 
measure, so ignore this test")
 @Test
 public void testCiCube() {
 CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig());

[35/39] kylin git commit: Fix bug in static init of DataType to pass ExtendedColumnSerializerTest UT

[lidong]

Fix bug in static init of DataType to pass ExtendedColumnSerializerTest UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/570ab42a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/570ab42a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/570ab42a

Branch: refs/heads/master-hbase0.98
Commit: 570ab42a9b7597ce85b17202b67cd0ea5403cba4
Parents: df3ecd3
Author: kangkaisen 
Authored: Wed Feb 8 21:25:37 2017 +0800
Committer: kangkaisen 
Committed: Wed Feb 8 21:38:00 2017 +0800

--
 .../main/java/org/apache/kylin/metadata/datatype/DataType.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/570ab42a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index b726c5f..d3756b8 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -114,7 +114,8 @@ public class DataType implements Serializable {
 public static final DataType ANY = DataType.getType("any");
 
 static {
-MeasureTypeFactory.init();
+//to ensure the MeasureTypeFactory class has initialized
+MeasureTypeFactory.getUDAFs();
 }
 
 public static DataType getType(String type) {

[08/39] kylin git commit: KYLIN-2361 add Tomcat8 ordered class loader

[lidong]

KYLIN-2361 add Tomcat8 ordered class loader


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a3bd71c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a3bd71c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a3bd71c

Branch: refs/heads/master-hbase0.98
Commit: 9a3bd71c8e5ce9dc13e38560efc556dc862819a1
Parents: 2b60ac6
Author: Billy Liu 
Authored: Sat Feb 4 11:41:49 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 11:42:02 2017 +0800

--
 .../kylin/ext/CustomizedWebappClassloader.java  |   4 +-
 .../kylin/ext/OrderedWebResourceRoot.java   | 286 +++
 .../kylin/ext/WebappOrderedClassLoader.java |  66 +
 3 files changed, 353 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index f241865..bbf4053 100644
--- 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -18,14 +18,12 @@
 
 package org.apache.kylin.ext;
 
-import org.apache.catalina.loader.ParallelWebappClassLoader;
-
 /**
  * simple extension to standard ParallelWebappClassLoader
  * the only difference is that CustomizedWebappClassloader is able to delegate 
more packages
  * to parent classloaders
  */
-public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
+public class CustomizedWebappClassloader extends WebappOrderedClassLoader {
 /**
  * Set of package names which are not allowed to be loaded from a webapp
  * class loader without delegating first.

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
new file mode 100644
index 000..9784bd8
--- /dev/null
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.ext;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.catalina.Context;
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.LifecycleListener;
+import org.apache.catalina.LifecycleState;
+import org.apache.catalina.TrackedWebResource;
+import org.apache.catalina.WebResource;
+import org.apache.catalina.WebResourceRoot;
+import org.apache.catalina.WebResourceSet;
+
+public class OrderedWebResourceRoot implements WebResourceRoot {
+
+private static final String WEB_INF_LIB_PATH = "/WEB-INF/lib";
+
+private static final Comparator WEB_RESOURCE_COMPARATOR = new 
Comparator() {
+@Override
+public int compare(WebResource o1, WebResource o2) {
+return o1.getName().compareTo(o2.getName());
+}
+};
+
+private WebResourceRoot delegate;
+
+public OrderedWebResourceRoot(WebResourceRoot delegate) {
+this.delegate = delegate;
+}
+
+@Override
+public WebResource[] listResources(String path) {
+WebResource[] webResources = delegate.listResources(path);
+
+if (WEB_INF_LIB_PATH.equals(path)) {
+Arrays.sort(webResources, WEB_RESOURCE_COMPARATOR);
+}
+
+return webResources;
+}
+
+@Override
+public void addLifecycleListener(LifecycleListener listener) {
+

[05/39] kylin git commit: KYLIN-2418 drop unused useSandbox

[lidong]

KYLIN-2418 drop unused useSandbox


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/546f88f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/546f88f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/546f88f3

Branch: refs/heads/master-hbase0.98
Commit: 546f88f38ca017773c17ccaf2cf4812b0c7c4666
Parents: 0582512
Author: Yang Li 
Authored: Mon Jan 30 09:07:29 2017 +0800
Committer: Yang Li 
Committed: Tue Jan 31 08:11:41 2017 +0800

--
 .../kylin/common/persistence/FileResourceStore.java   |  4 
 .../org/apache/kylin/common/persistence/ResourceTool.java | 10 ++
 kylin-it/pom.xml  |  6 --
 pom.xml   |  8 
 .../apache/kylin/storage/hbase/HBaseResourceStore.java|  5 +
 5 files changed, 15 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index dcd3f38..3e012f5 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -160,4 +160,8 @@ public class FileResourceStore extends ResourceStore {
 return new File(root, resPath);
 }
 
+@Override
+public String toString() {
+return root.getAbsolutePath();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index b3aac09..6ba68ae 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -133,22 +133,24 @@ public class ResourceTool {
 ResourceStore src = ResourceStore.getStore(srcConfig);
 ResourceStore dst = ResourceStore.getStore(dstConfig);
 
+logger.info("Copy from {} to {}", src, dst);
+
 copyR(src, dst, path);
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, 
List paths) throws IOException {
 ResourceStore src = ResourceStore.getStore(srcConfig);
 ResourceStore dst = ResourceStore.getStore(dstConfig);
+
+logger.info("Copy from {} to {}", src, dst);
+
 for (String path : paths) {
 copyR(src, dst, path);
 }
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) 
throws IOException {
-
-ResourceStore src = ResourceStore.getStore(srcConfig);
-ResourceStore dst = ResourceStore.getStore(dstConfig);
-copyR(src, dst, "/");
+copy(srcConfig, dstConfig, "/");
 }
 
 public static void copyR(ResourceStore src, ResourceStore dst, String 
path) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index f88db9f..9662806 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -272,10 +272,6 @@
 
 
 
-useSandbox
-true
-
-
 log4j.configuration
 
file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 
@@ -298,7 +294,6 @@
 test
 java
 
--DuseSandbox=true
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
 
-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
@@ -320,7 +315,6 @@
 test
 java
  

[22/39] kylin git commit: KYLIN-2406 fix model match rules when having hanging OLAPTablescan

[lidong]

KYLIN-2406 fix model match rules when having hanging OLAPTablescan

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f444681
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f444681
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f444681

Branch: refs/heads/master-hbase0.98
Commit: 1f444681d30ed5f31997c2182cce0f388fef0347
Parents: ef0fc86
Author: etherge 
Authored: Sat Feb 4 11:31:58 2017 +0800
Committer: lidongsjtu 
Committed: Mon Feb 6 17:06:55 2017 +0800

--
 .../main/java/org/apache/kylin/query/routing/ModelChooser.java| 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1f444681/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java 
b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index f979f7b..507b371 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -106,6 +106,9 @@ public class ModelChooser {
 // one lookup table
 String modelAlias = 
model.findFirstTable(firstTable.getTableIdentity()).getAlias();
 matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias);
+} else if (ctx.joins.size() != ctx.allTableScans.size() - 1) {
+// has hanging tables
+throw new NoRealizationFoundException("Please adjust the 
sequence of join tables and put subquery or temporary table after lookup 
tables. " + toErrorMsg(ctx));
 } else {
 // normal big joins
 if (ctx.joinsTree == null) {

[37/39] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/4e41c363/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
--
diff --git 
a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.ResourceStore;
@@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI {
 Serializer projectSerializer = new 
JsonSerializer(ProjectInstance.class);
 ProjectInstance project = store.getResource(projectResPath, 
ProjectInstance.class, projectSerializer);
 String projUUID = project.getUuid();
-Table aclHtable = null;
+HTableInterface aclHtable = null;
 try {
-aclHtable = 
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix()
 + "_acl"));
+aclHtable = 
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix()
 + "_acl");
 
 // cube acl
 Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI {
 aclHtable.put(put);
 }
 }
+aclHtable.flushCommits();
 } finally {
 IOUtils.closeQuietly(aclHtable);
 }

[28/39] kylin git commit: KYLIN-2434 minor fix

[lidong]

KYLIN-2434 minor fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eee9ecbc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eee9ecbc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eee9ecbc

Branch: refs/heads/master-hbase0.98
Commit: eee9ecbcb031862ed316f630933bb320cf93a9d1
Parents: f62465c
Author: lidongsjtu 
Authored: Tue Feb 7 17:37:21 2017 +0800
Committer: lidongsjtu 
Committed: Tue Feb 7 17:37:21 2017 +0800

--
 .../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eee9ecbc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 071806c..317d2bd 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,12 +150,12 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 setupClasspath(sc, confPath);
 HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
 
+System.setProperty(KylinConfig.KYLIN_CONF, confPath);
 final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
 
 HiveContext sqlContext = new HiveContext(sc.sc());
 final DataFrame intermediateTable = 
sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + 
hiveTable);
 
-System.setProperty(KylinConfig.KYLIN_CONF, confPath);
 final CubeInstance cubeInstance = 
CubeManager.getInstance(envConfig).getCube(cubeName);
 final CubeDesc cubeDesc = cubeInstance.getDescriptor();
 final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

[27/39] kylin git commit: KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing

[lidong]

KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f62465cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f62465cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f62465cc

Branch: refs/heads/master-hbase0.98
Commit: f62465cc5a5fe0f90142c0606c0c75e8c721bf0e
Parents: b32cc95
Author: lidongsjtu 
Authored: Tue Feb 7 17:31:07 2017 +0800
Committer: lidongsjtu 
Committed: Tue Feb 7 17:31:14 2017 +0800

--
 .../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f62465cc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 8892a73..071806c 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,11 +150,12 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 setupClasspath(sc, confPath);
 HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
 
+final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
+
 HiveContext sqlContext = new HiveContext(sc.sc());
-final DataFrame intermediateTable = sqlContext.table(hiveTable);
+final DataFrame intermediateTable = 
sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + 
hiveTable);
 
 System.setProperty(KylinConfig.KYLIN_CONF, confPath);
-final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
 final CubeInstance cubeInstance = 
CubeManager.getInstance(envConfig).getCube(cubeName);
 final CubeDesc cubeDesc = cubeInstance.getDescriptor();
 final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

[20/39] kylin git commit: Merge branch 'KYLIN-2361'

[lidong]

Merge branch 'KYLIN-2361'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4047e8dc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4047e8dc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4047e8dc

Branch: refs/heads/master-hbase0.98
Commit: 4047e8dc5bf8aad7c8db79abb5ef2c3be15cd622
Parents: 1925284 9a3bd71
Author: Billy Liu 
Authored: Mon Feb 6 14:37:46 2017 +0800
Committer: Billy Liu 
Committed: Mon Feb 6 14:37:46 2017 +0800

--
 build/script/download-tomcat.sh |   8 +-
 pom.xml |   2 +-
 .../java/org/apache/kylin/rest/DebugTomcat.java |  16 +-
 .../kylin/ext/CustomizedWebappClassloader.java  |   9 +-
 .../kylin/ext/OrderedWebResourceRoot.java   | 286 +++
 .../kylin/ext/WebappOrderedClassLoader.java |  66 +
 6 files changed, 369 insertions(+), 18 deletions(-)
--

[11/39] kylin git commit: KYLIN-2423 Model should always include PK/FK as dimensions

[lidong]

KYLIN-2423 Model should always include PK/FK as dimensions


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/855301dc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/855301dc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/855301dc

Branch: refs/heads/master-hbase0.98
Commit: 855301dc7fc83ece4f03dd981f3184bb46a97105
Parents: 56c7aa5
Author: Li Yang 
Authored: Sat Feb 4 15:19:31 2017 +0800
Committer: Li Yang 
Committed: Sat Feb 4 16:12:42 2017 +0800

--
 .../kylin/metadata/model/DataModelDesc.java | 76 
 .../kylin/metadata/model/JoinTableDesc.java |  7 +-
 .../apache/kylin/metadata/model/JoinsTree.java  | 11 +--
 .../metadata/model/ModelDimensionDesc.java  |  2 +
 .../model_desc/ut_large_dimension_number.json   |  2 +-
 5 files changed, 90 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/855301dc/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index cc599a4..86eea55 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -20,15 +20,18 @@ package org.apache.kylin.metadata.model;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.model.JoinsTree.Chain;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -294,6 +297,11 @@ public class DataModelDesc extends RootPersistentEntity {
 initJoinsTree();
 initDimensionsAndMetrics();
 initPartitionDesc();
+
+boolean reinit = validate();
+if (reinit) { // model slightly changed by validate() and must init() 
again
+init(config, tables);
+}
 }
 
 private void initJoinTablesForUpgrade() {
@@ -443,6 +451,74 @@ public class DataModelDesc extends RootPersistentEntity {
 joinsTree = new JoinsTree(rootFactTableRef, joins);
 }
 
+private boolean validate() {
+Set mcols = new HashSet<>();
+for (String m : metrics) {
+mcols.add(findColumn(m));
+}
+
+// validate no dup between dimensions/metrics
+for (ModelDimensionDesc dim : dimensions) {
+String table = dim.getTable();
+for (String c : dim.getColumns()) {
+TblColRef dcol = findColumn(table, c);
+if (mcols.contains(dcol))
+throw new IllegalStateException(dcol + " cannot be both 
dimension and metrics at the same time in " + this);
+}
+}
+
+// validate PK/FK are in dimensions
+boolean pkfkDimAmended = false;
+for (Chain chain : joinsTree.tableChains.values()) {
+pkfkDimAmended = validatePkFkDim(chain.join, mcols) || 
pkfkDimAmended;
+}
+return pkfkDimAmended;
+}
+
+private boolean validatePkFkDim(JoinDesc join, Set mcols) {
+if (join == null)
+return false;
+
+boolean pkfkDimAmended = false;
+
+for (TblColRef c : join.getForeignKeyColumns()) {
+if (!mcols.contains(c)) {
+pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended;
+}
+}
+for (TblColRef c : join.getPrimaryKeyColumns()) {
+if (!mcols.contains(c)) {
+pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended;
+}
+}
+return pkfkDimAmended;
+}
+
+private boolean validatePkFkDim(TblColRef c) {
+String t = c.getTableAlias();
+ModelDimensionDesc dimDesc = null;
+for (ModelDimensionDesc dim : dimensions) {
+if (dim.getTable().equals(t)) {
+dimDesc = dim;
+break;
+}
+}
+
+if (dimDesc == null) {
+dimDesc = new ModelDimensionDesc();
+dimDesc.setTable(t);
+dimDesc.setColumns(new String[0]);
+}
+
+if (ArrayUtils.contains(dimDesc.getColumns(), c.getName()) == false) {
+   

[18/39] kylin git commit: Update sample cube to pass latest validation

[lidong]

Update sample cube to pass latest validation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9cd6c707
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9cd6c707
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9cd6c707

Branch: refs/heads/master-hbase0.98
Commit: 9cd6c7075839e41a0f2ff71334343613ce503cd3
Parents: 663a6f9
Author: shaofengshi 
Authored: Sun Feb 5 11:31:13 2017 +0800
Committer: shaofengshi 
Committed: Sun Feb 5 11:31:13 2017 +0800

--
 examples/sample_cube/template/model_desc/kylin_sales_model.json | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9cd6c707/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index cce360f..cfe6845 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -76,9 +76,7 @@
   } ],
   "metrics": [
 "PRICE",
-"ITEM_COUNT",
-"SELLER_ID",
-"LSTG_FORMAT_NAME"
+"ITEM_COUNT"
   ],
   "last_modified" : 1422435345362,
   "fact_table" : "DEFAULT.KYLIN_SALES",

[26/39] kylin git commit: KYLIN-2433 Fix NPE in MergeCuboidMapper

[lidong]

KYLIN-2433 Fix NPE in MergeCuboidMapper


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b32cc954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b32cc954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b32cc954

Branch: refs/heads/master-hbase0.98
Commit: b32cc9545a76657570c4a3353469ded43892c772
Parents: 6d11dd1
Author: kangkaisen 
Authored: Fri Feb 3 14:00:50 2017 +0800
Committer: kangkaisen 
Committed: Tue Feb 7 17:20:30 2017 +0800

--
 .../apache/kylin/engine/mr/steps/MergeCuboidMapper.java   | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b32cc954/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 047e2b1..acf1403 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -129,6 +129,11 @@ public class MergeCuboidMapper extends KylinMapper {
 List columns = 
measureType.getColumnsNeedDictionary(measureDesc.getFunction());
 boolean needReEncode = false;
 for (TblColRef col : columns) {
+//handle the column that all records is null
+if (sourceCubeSegment.getDictionary(col) == null) {
+continue;
+}
+
 if 
(!sourceCubeSegment.getDictionary(col).equals(mergedCubeSegment.getDictionary(col)))
 {
 oldDicts.put(col, sourceCubeSegment.getDictionary(col));
 newDicts.put(col, mergedCubeSegment.getDictionary(col));
@@ -249,6 +254,11 @@ public class MergeCuboidMapper extends KylinMapper {
 }
 
 private Boolean checkNeedMerging(TblColRef col) throws IOException {
+//handle the column that all records is null
+if (sourceCubeSegment.getDictionary(col) == null) {
+return false;
+}
+
 Boolean ret = dimensionsNeedDict.get(col);
 if (ret != null)
 return ret;

[01/39] kylin git commit: KYLIN-2374 code review [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master-hbase0.98 9323c7c2b -> 4e41c3637 (forced update)


KYLIN-2374 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eae37ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eae37ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eae37ef

Branch: refs/heads/master-hbase0.98
Commit: 5eae37ef18ca51027c6bb2cfd3410fefc7982f2a
Parents: a2a59c4
Author: shaofengshi 
Authored: Thu Jan 26 09:55:48 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 09:55:48 2017 +0800

--
 build/conf/kylin.properties |  3 +-
 build/deploy/spark-defaults.conf|  1 -
 .../apache/kylin/common/KylinConfigBase.java|  8 --
 .../kylin/common/persistence/ResourceStore.java |  3 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  2 +-
 .../ExtendedColumnMeasureType.java  |  8 +-
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 36 +++-
 .../kylin/storage/hbase/HBaseResourceStore.java |  3 +-
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 90 +++-
 9 files changed, 97 insertions(+), 57 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index eceb886..43ea17d 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -211,8 +211,9 @@ kylin.engine.spark-conf.spark.executor.memory=4G
 kylin.engine.spark-conf.spark.executor.cores=4
 kylin.engine.spark-conf.spark.executor.instances=8
 kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
-kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.eventLog.enabled=true
 kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
 ## manually upload spark-assembly jar to HDFS and then set this property will 
avoid repeatedly uploading jar at runtime
 
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/deploy/spark-defaults.conf
--
diff --git a/build/deploy/spark-defaults.conf b/build/deploy/spark-defaults.conf
index 36c0ab3..78a4bc9 100644
--- a/build/deploy/spark-defaults.conf
+++ b/build/deploy/spark-defaults.conf
@@ -1,5 +1,4 @@
 spark.yarn.submit.file.replication=1
-spark.eventLog.enabled=true
 spark.yarn.max.executor.failures=3
 spark.driver.extraJavaOptions=-Dhdp.version=current
 spark.yarn.am.extraJavaOptions=-Dhdp.version=current

http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5932197..b1acbbf 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,14 +193,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
-public String getRawHdfsWorkingDirectory() {
-String root = getRequired("kylin.env.hdfs-working-dir");
-if (!root.endsWith("/")) {
-root += "/";
-}
-return root;
-}
-
 // 

 // METADATA
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 25a0801..c441618 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -63,6 +63,9 @@ abstract 

[36/39] kylin git commit: KYLIN-2222 web ui uses rest api to decide which dim encoding is valid for different typed columns

[lidong]

KYLIN- web ui uses rest api to decide which dim encoding is valid for 
different typed columns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/722efb82
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/722efb82
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/722efb82

Branch: refs/heads/master-hbase0.98
Commit: 722efb82357e0ebcf7853a813272bd960044dd52
Parents: 570ab42
Author: Hongbin Ma 
Authored: Wed Feb 8 21:41:41 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Feb 8 21:41:57 2017 +0800

--
 .../rest/controller/EncodingController.java | 73 
 .../kylin/rest/service/EncodingService.java | 54 +++
 2 files changed, 127 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/722efb82/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
new file mode 100644
index 000..2f532e2
--- /dev/null
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.metadata.datatype.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import org.apache.kylin.rest.service.EncodingService;
+
+@Controller
+@RequestMapping(value = "/encodings")
+public class EncodingController extends BasicController {
+
+private static final Logger logger = 
LoggerFactory.getLogger(EncodingController.class);
+
+@Autowired
+private EncodingService encodingService;
+
+/**
+ * Get valid encodings for the datatype, if no datatype parameter, return 
all encodings.
+ *
+ * @return suggestion map
+ */
+@RequestMapping(value = "valid_encodings", method = { RequestMethod.GET })
+@ResponseBody
+public Map getValidEncodings() {
+
+Set allDatatypes = Sets.newHashSet();
+allDatatypes.addAll(DataType.DATETIME_FAMILY);
+allDatatypes.addAll(DataType.INTEGER_FAMILY);
+allDatatypes.addAll(DataType.NUMBER_FAMILY);
+allDatatypes.addAll(DataType.STRING_FAMILY);
+
+Map datatypeValidEncodings = Maps.newHashMap();
+for (String dataTypeStr : allDatatypes) {
+datatypeValidEncodings.put(dataTypeStr, 
encodingService.getValidEncodings(DataType.getType(dataTypeStr)));
+}
+
+Map ret = Maps.newHashMap();
+ret.put("code", "000");
+ret.put("data", datatypeValidEncodings);
+return ret;
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/722efb82/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
new file mode 100644
index 000..7d7d016
--- /dev/null
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for 

[06/39] kylin git commit: minor, stablize DefaultSchedulerTest

[lidong]

minor, stablize DefaultSchedulerTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a058bfb8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a058bfb8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a058bfb8

Branch: refs/heads/master-hbase0.98
Commit: a058bfb8b0490fe36b7fe4da026028411ed208a5
Parents: 546f88f
Author: Li Yang 
Authored: Fri Feb 3 13:43:56 2017 +0800
Committer: Li Yang 
Committed: Fri Feb 3 13:43:56 2017 +0800

--
 .../job/impl/threadpool/BaseSchedulerTest.java  | 36 ++--
 1 file changed, 26 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a058bfb8/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index fdf5252..1ada9a1 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -31,11 +31,15 @@ import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.lock.MockJobLock;
 import org.junit.After;
 import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public abstract class BaseSchedulerTest extends LocalFileMetadataTestCase {
 
+private static final Logger logger = 
LoggerFactory.getLogger(BaseSchedulerTest.class);
+
 private DefaultScheduler scheduler;
 
 protected ExecutableManager jobService;
@@ -70,19 +74,31 @@ public abstract class BaseSchedulerTest extends 
LocalFileMetadataTestCase {
 }
 
 protected void waitForJobFinish(String jobId) {
-while (true) {
-AbstractExecutable job = jobService.getJob(jobId);
-final ExecutableState status = job.getStatus();
-if (status == ExecutableState.SUCCEED || status == 
ExecutableState.ERROR || status == ExecutableState.STOPPED || status == 
ExecutableState.DISCARDED) {
-break;
-} else {
-try {
-Thread.sleep(5000);
-} catch (InterruptedException e) {
-e.printStackTrace();
+int error = 0;
+final int errorLimit = 3;
+
+while (error < errorLimit) {
+try {
+Thread.sleep(2000);
+} catch (InterruptedException e) {
+e.printStackTrace();
+}
+
+try {
+AbstractExecutable job = jobService.getJob(jobId);
+ExecutableState status = job.getStatus();
+if (status == ExecutableState.SUCCEED || status == 
ExecutableState.ERROR || status == ExecutableState.STOPPED || status == 
ExecutableState.DISCARDED) {
+break;
 }
+} catch (Exception ex) {
+logger.error("", ex);
+error++;
 }
 }
+
+if (error >= errorLimit) {
+throw new RuntimeException("waitForJobFinish() encounters 
exceptions, see logs above");
+}
 }
 
 protected void waitForJobStatus(String jobId, ExecutableState state, long 
interval) {

[16/39] kylin git commit: KYLIN-2424 Optimize the integration test's performance

[lidong]

KYLIN-2424 Optimize the integration test's performance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d83c80f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d83c80f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d83c80f

Branch: refs/heads/master-hbase0.98
Commit: 5d83c80fbc0d6c3db434368cc8c29d786eebcd94
Parents: 6f0bc1c
Author: shaofengshi 
Authored: Sat Feb 4 21:37:05 2017 +0800
Committer: shaofengshi 
Committed: Sat Feb 4 21:37:05 2017 +0800

--
 .../apache/kylin/metadata/model/ColumnDesc.java |  8 +++
 .../spark/SparkBatchCubingJobBuilder2.java  |  2 +-
 .../table/DEFAULT.STREAMING_TABLE.json  |  9 +---
 .../localmeta/table/DEFAULT.TEST_ACCOUNT.json   |  6 +++--
 .../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json  |  9 +---
 .../localmeta/table/DEFAULT.TEST_COUNTRY.json   |  6 +++--
 .../table/DEFAULT.TEST_KYLIN_FACT.json  | 21 --
 .../localmeta/table/DEFAULT.TEST_ORDER.json |  6 +++--
 .../localmeta/table/EDW.TEST_CAL_DT.json| 15 -
 .../table/EDW.TEST_SELLER_TYPE_DIM.json |  6 +++--
 .../table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json   |  6 +++--
 .../localmeta/table/EDW.TEST_SITES.json |  9 +---
 .../java/org/apache/kylin/query/H2Database.java | 23 
 .../org/apache/kylin/query/KylinTestBase.java   |  4 +++-
 14 files changed, 97 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 7105ede..e5b51e4 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -49,6 +49,10 @@ public class ColumnDesc implements Serializable {
 @JsonInclude(JsonInclude.Include.NON_NULL)
 private String dataGen;
 
+@JsonProperty("index")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+private String index;
+
 // parsed from data type
 private DataType type;
 private DataType upgradedType;
@@ -157,6 +161,10 @@ public class ColumnDesc implements Serializable {
 return dataGen;
 }
 
+public String getIndex() {
+return index;
+}
+
 public void init(TableDesc table) {
 this.table = table;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 76b73b6..327d215 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -77,7 +77,7 @@ public class SparkBatchCubingJobBuilder2 extends 
BatchCubingJobBuilder2 {
 try {
 return ClassUtil.findContainingJar(Class.forName(className));
 } catch (ClassNotFoundException e) {
-logger.error("failed to locate jar for class " + className, e);
+logger.warn("failed to locate jar for class " + className + ", 
ignore it", e);
 }
 
 return "";

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
index f28683f..d67cbe5 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
@@ -5,17 +5,20 @@
 {
   "id": "1",
   "name": "minute_start",
-  "datatype": "timestamp"
+  "datatype": "timestamp",
+  "index": "T"
 },
 {
   "id": "2",
   "name": "hour_start",
-  "datatype": "timestamp"
+  "datatype": "timestamp",
+  "index": "T"
 },
 {
   "id": "3",
   "name": "day_start",
-  "datatype": "date"
+  "datatype": "date",
+  "index": "T"
 },
 {
   "id": "4",

[39/39] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

[lidong]

KYLIN-2307 Create a branch for master with HBase 0.98 API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4e41c363
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4e41c363
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4e41c363

Branch: refs/heads/master-hbase0.98
Commit: 4e41c36370eb458307998acd09e140667d33b7ea
Parents: 722efb8
Author: lidongsjtu 
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: lidongsjtu 
Committed: Thu Feb 9 10:23:07 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |  25 
 dev-support/test_all_against_hdp_2_4_0_0_169.sh |  25 
 .../sandbox/capacity-scheduler.xml  |  17 ++-
 examples/test_case_data/sandbox/core-site.xml   |  28 +---
 examples/test_case_data/sandbox/hbase-site.xml  | 119 +
 examples/test_case_data/sandbox/hdfs-site.xml   |  84 +---
 examples/test_case_data/sandbox/hive-site.xml   |  89 +
 examples/test_case_data/sandbox/mapred-site.xml |  57 +++--
 examples/test_case_data/sandbox/yarn-site.xml   | 127 +++
 .../kylin/provision/BuildCubeWithEngine.java|  17 +--
 pom.xml | 122 +-
 .../kylin/rest/security/AclHBaseStorage.java|   4 +-
 .../rest/security/MockAclHBaseStorage.java  |   8 +-
 .../apache/kylin/rest/security/MockHTable.java  |  95 +++---
 .../rest/security/RealAclHBaseStorage.java  |   9 +-
 .../apache/kylin/rest/service/AclService.java   |  25 ++--
 .../apache/kylin/rest/service/CubeService.java  |  35 +++--
 .../apache/kylin/rest/service/QueryService.java |  24 ++--
 .../apache/kylin/rest/service/UserService.java  |  17 +--
 .../kylin/storage/hbase/HBaseConnection.java|  44 +++
 .../kylin/storage/hbase/HBaseResourceStore.java |  31 +++--
 .../storage/hbase/cube/SimpleHBaseStore.java|  20 +--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  13 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |   4 +-
 .../storage/hbase/steps/CubeHTableUtil.java |  16 +--
 .../storage/hbase/steps/DeprecatedGCStep.java   |  24 ++--
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  |  23 ++--
 .../storage/hbase/util/CleanHtableCLI.java  |  12 +-
 .../storage/hbase/util/CubeMigrationCLI.java|  37 +++---
 .../hbase/util/CubeMigrationCheckCLI.java   |  17 +--
 .../hbase/util/DeployCoprocessorCLI.java|  27 ++--
 .../hbase/util/ExtendCubeToHybridCLI.java   |   8 +-
 .../hbase/util/GridTableHBaseBenchmark.java |  34 ++---
 .../kylin/storage/hbase/util/HBaseClean.java|  18 ++-
 .../hbase/util/HBaseRegionSizeCalculator.java   |  35 +++--
 .../kylin/storage/hbase/util/HBaseUsage.java|   9 +-
 .../storage/hbase/util/HbaseStreamingInput.java |  30 ++---
 .../hbase/util/HtableAlterMetadataCLI.java  |   9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java |  19 +--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  |  15 +--
 .../kylin/storage/hbase/util/RowCounterCLI.java |  11 +-
 .../storage/hbase/util/StorageCleanupJob.java   |  20 ++-
 .../storage/hbase/util/UpdateHTableHostCLI.java |  17 +--
 tool/pom.xml|  10 --
 .../org/apache/kylin/tool/CubeMigrationCLI.java |  19 +--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |   8 +-
 48 files changed, 596 insertions(+), 877 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4e41c363/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install 

[07/39] kylin git commit: KYLIN-2419 rollback KYLIN-2292

[lidong]

KYLIN-2419 rollback KYLIN-2292


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7611338b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7611338b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7611338b

Branch: refs/heads/master-hbase0.98
Commit: 7611338b5f022d216f5c9564a13c161374751adf
Parents: a058bfb
Author: Li Yang 
Authored: Sat Feb 4 11:03:25 2017 +0800
Committer: Li Yang 
Committed: Sat Feb 4 11:03:25 2017 +0800

--
 .../adapter/enumerable/EnumerableWindow.java| 981 ---
 .../calcite/adapter/enumerable/PhysType.java| 209 
 .../adapter/enumerable/PhysTypeImpl.java| 654 -
 3 files changed, 1844 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7611338b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
deleted file mode 100644
index 216b07c..000
--- 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
+++ /dev/null
@@ -1,981 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.adapter.enumerable;
-
-import org.apache.calcite.adapter.enumerable.impl.WinAggAddContextImpl;
-import org.apache.calcite.adapter.enumerable.impl.WinAggResetContextImpl;
-import org.apache.calcite.adapter.enumerable.impl.WinAggResultContextImpl;
-import org.apache.calcite.adapter.java.JavaTypeFactory;
-import org.apache.calcite.linq4j.tree.BinaryExpression;
-import org.apache.calcite.linq4j.tree.BlockBuilder;
-import org.apache.calcite.linq4j.tree.BlockStatement;
-import org.apache.calcite.linq4j.tree.DeclarationStatement;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.ParameterExpression;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.linq4j.tree.Statement;
-import org.apache.calcite.linq4j.tree.Types;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.prepare.CalcitePrepareImpl;
-import org.apache.calcite.rel.RelFieldCollation;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.core.Window;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexWindowBound;
-import org.apache.calcite.runtime.SortedMultiMap;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.util.BuiltInMethod;
-import org.apache.calcite.util.Pair;
-import org.apache.calcite.util.Util;
-
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
-
-import java.lang.reflect.Modifier;
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-
-/*
- * OVERRIDE POINT: patching CALCITE-1540 on calcite 1.8.0
- */
-
-/** Implementation of {@link org.apache.calcite.rel.core.Window} in
- * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention 
enumerable calling convention}. */
-public class EnumerableWindow extends Window implements EnumerableRel {
-/** Creates an EnumerableWindowRel. */
-EnumerableWindow(RelOptCluster cluster, RelTraitSet traits, RelNode child,
-List constants, RelDataType rowType, List 
groups) {
-super(cluster, traits, child, 

[32/39] kylin git commit: KYLIN-2377 Add kylin client query timeout

[lidong]

KYLIN-2377 Add kylin client query timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8263752a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8263752a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8263752a

Branch: refs/heads/master-hbase0.98
Commit: 8263752a499158342e0588fda851a4006e8b1669
Parents: 0dcce15
Author: kangkaisen 
Authored: Tue Jan 10 14:02:18 2017 +0800
Committer: kangkaisen 
Committed: Wed Feb 8 15:51:30 2017 +0800

--
 .../org/apache/kylin/common/KylinConfigBase.java|  4 
 .../java/org/apache/kylin/cube/CubeInstance.java|  1 +
 .../kylin/metadata/realization/IRealization.java|  5 -
 .../org/apache/kylin/storage/StorageContext.java| 16 +++-
 .../storage/gtrecord/GTCubeStorageQueryBase.java|  2 ++
 .../gtrecord/SequentialCubeTupleIterator.java   |  5 +
 .../apache/kylin/storage/hybrid/HybridInstance.java |  1 +
 7 files changed, 32 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index fe15b1e..ebd9dfc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -877,6 +877,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return udfMap;
 }
 
+public int getQueryTimeoutSeconds() {
+return 
Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+}
+
 // 

 // SERVER
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 1d60575..fb9a7a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return sizeRecordSize;
 }
 
+@Override
 public KylinConfig getConfig() {
 return config;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index a0243f4..aafc0f0 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
@@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware {
 public DataModelDesc getModel();
 
 public Set getAllColumns();
-
+
 public Set getAllColumnDescs();
 
 public List getAllDimensions();
@@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware {
 public long getDateRangeEnd();
 
 public boolean supportsLimitPushDown();
+
+public KylinConfig getConfig();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java 
b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index ec46f83..ab0ea73 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -41,6 +41,7 @@ public class StorageContext {
 private int finalPushDownLimit = Integer.MAX_VALUE;
 private boolean hasSort = false;
 

[19/39] kylin git commit: KYLIN-2426 fix hardcode path

[lidong]

KYLIN-2426 fix hardcode path

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/19252848
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/19252848
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/19252848

Branch: refs/heads/master-hbase0.98
Commit: 19252848e27423edeb6f3ebed6dea453db1904e9
Parents: 9cd6c70
Author: etherge 
Authored: Mon Feb 6 11:38:42 2017 +0800
Committer: lidongsjtu 
Committed: Mon Feb 6 13:48:38 2017 +0800

--
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 39 +++-
 1 file changed, 21 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/19252848/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index ec12722..aa5a104 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -18,6 +18,9 @@
 
 package org.apache.kylin.storage.hdfs;
 
+import static junit.framework.TestCase.assertTrue;
+
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.kylin.common.KylinConfig;
@@ -28,18 +31,18 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import static junit.framework.TestCase.assertTrue;
-
 public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
 
 KylinConfig kylinConfig;
 FileSystem fs;
+String workingDir;
 
 @Before
 public void setup() throws Exception {
 this.createTestMetadata();
 kylinConfig = KylinConfig.getInstanceFromEnv();
 fs = HadoopUtil.getWorkingFileSystem();
+workingDir = getHdfsWorkingDirWithoutScheme(kylinConfig);
 }
 
 @After
@@ -47,32 +50,33 @@ public class ITHDFSResourceStoreTest extends 
HBaseMetadataTestCase {
 this.cleanupTestMetadata();
 }
 
+private String getHdfsWorkingDirWithoutScheme(KylinConfig kylinConfig) {
+String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
+int thirdIndex = StringUtils.ordinalIndexOf(hdfsWorkingDir, "/", 3);
+int fourthIndex = StringUtils.ordinalIndexOf(hdfsWorkingDir, "/", 5);
+return hdfsWorkingDir.substring(thirdIndex, fourthIndex);
+}
+
 @Test
 public void testBasic() throws Exception {
-String oldUrl = kylinConfig.getMetadataUrl();
-String path = "/kylin/kylin_metadata/metadata";
-kylinConfig.setProperty("kylin.metadata.url", path + "@hdfs");
-HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
-ResourceStoreTest.testAStore(store);
-kylinConfig.setProperty("kylin.metadata.url", oldUrl);
-assertTrue(fs.exists(new Path(path)));
+String path = workingDir + "/metadata_test1";
+doTestWithPath(path);
 }
 
 @Test
 public void testQalifiedName() throws Exception {
-String oldUrl = kylinConfig.getMetadataUrl();
-String path = "hdfs:///kylin/kylin_metadata/metadata_test1";
-kylinConfig.setProperty("kylin.metadata.url", path + "@hdfs");
-HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
-ResourceStoreTest.testAStore(store);
-kylinConfig.setProperty("kylin.metadata.url", oldUrl);
-assertTrue(fs.exists(new Path(path)));
+String path = "hdfs://" + workingDir + "/metadata_test2";
+doTestWithPath(path);
 }
 
 @Test
 public void testFullQalifiedName() throws Exception {
+String path = fs.getUri() + workingDir + "/metadata_test3";
+doTestWithPath(path);
+}
+
+private void doTestWithPath(String path) throws Exception {
 String oldUrl = kylinConfig.getMetadataUrl();
-String path = 
"hdfs://sandbox.hortonworks.com:8020/kylin/kylin_metadata/metadata_test2";
 kylinConfig.setProperty("kylin.metadata.url", path + "@hdfs");
 HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
 ResourceStoreTest.testAStore(store);
@@ -80,5 +84,4 @@ public class ITHDFSResourceStoreTest extends 
HBaseMetadataTestCase {
 assertTrue(fs.exists(new Path(path)));
 }
 
-
 }

[29/39] kylin git commit: KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob

[lidong]

KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b860adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b860adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b860adb

Branch: refs/heads/master-hbase0.98
Commit: 7b860adb0a23b53d6e2e40337f4317a3a2b067ba
Parents: eee9ecb
Author: kangkaisen 
Authored: Tue Feb 7 17:35:48 2017 +0800
Committer: kangkaisen 
Committed: Tue Feb 7 17:40:39 2017 +0800

--
 .../kylin/storage/hbase/steps/BulkLoadJob.java | 17 -
 1 file changed, 12 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b860adb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 1c05767..30616c5 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -55,11 +55,18 @@ public class BulkLoadJob extends AbstractHadoopJob {
 
 Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
 FsShell shell = new FsShell(conf);
-try {
-shell.run(new String[] { "-chmod", "-R", "777", input });
-} catch (Exception e) {
-logger.error("Couldn't change the file permissions ", e);
-throw new IOException(e);
+
+int exitCode = -1;
+int retryCount = 10;
+while (exitCode != 0 && retryCount >= 1) {
+exitCode = shell.run(new String[] { "-chmod", "-R", "777", input 
});
+retryCount--;
+Thread.sleep(5000);
+}
+
+if (exitCode != 0) {
+logger.error("Failed to change the file permissions: " + input);
+throw new IOException("Failed to change the file permissions: " + 
input);
 }
 
 String[] newArgs = new String[2];

kylin git commit: KYLIN-2222 web ui uses rest api to decide which dim encoding is valid for different typed columns

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 570ab42a9 -> 722efb823


KYLIN- web ui uses rest api to decide which dim encoding is valid for 
different typed columns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/722efb82
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/722efb82
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/722efb82

Branch: refs/heads/master
Commit: 722efb82357e0ebcf7853a813272bd960044dd52
Parents: 570ab42
Author: Hongbin Ma 
Authored: Wed Feb 8 21:41:41 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Feb 8 21:41:57 2017 +0800

--
 .../rest/controller/EncodingController.java | 73 
 .../kylin/rest/service/EncodingService.java | 54 +++
 2 files changed, 127 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/722efb82/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
new file mode 100644
index 000..2f532e2
--- /dev/null
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.metadata.datatype.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import org.apache.kylin.rest.service.EncodingService;
+
+@Controller
+@RequestMapping(value = "/encodings")
+public class EncodingController extends BasicController {
+
+private static final Logger logger = 
LoggerFactory.getLogger(EncodingController.class);
+
+@Autowired
+private EncodingService encodingService;
+
+/**
+ * Get valid encodings for the datatype, if no datatype parameter, return 
all encodings.
+ *
+ * @return suggestion map
+ */
+@RequestMapping(value = "valid_encodings", method = { RequestMethod.GET })
+@ResponseBody
+public Map getValidEncodings() {
+
+Set allDatatypes = Sets.newHashSet();
+allDatatypes.addAll(DataType.DATETIME_FAMILY);
+allDatatypes.addAll(DataType.INTEGER_FAMILY);
+allDatatypes.addAll(DataType.NUMBER_FAMILY);
+allDatatypes.addAll(DataType.STRING_FAMILY);
+
+Map datatypeValidEncodings = Maps.newHashMap();
+for (String dataTypeStr : allDatatypes) {
+datatypeValidEncodings.put(dataTypeStr, 
encodingService.getValidEncodings(DataType.getType(dataTypeStr)));
+}
+
+Map ret = Maps.newHashMap();
+ret.put("code", "000");
+ret.put("data", datatypeValidEncodings);
+return ret;
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/722efb82/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
new file mode 100644
index 000..7d7d016
--- /dev/null
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license 

kylin git commit: Fix bug in static init of DataType to pass ExtendedColumnSerializerTest UT

[kangkaisen]

Repository: kylin
Updated Branches:
  refs/heads/master df3ecd324 -> 570ab42a9


Fix bug in static init of DataType to pass ExtendedColumnSerializerTest UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/570ab42a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/570ab42a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/570ab42a

Branch: refs/heads/master
Commit: 570ab42a9b7597ce85b17202b67cd0ea5403cba4
Parents: df3ecd3
Author: kangkaisen 
Authored: Wed Feb 8 21:25:37 2017 +0800
Committer: kangkaisen 
Committed: Wed Feb 8 21:38:00 2017 +0800

--
 .../main/java/org/apache/kylin/metadata/datatype/DataType.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/570ab42a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index b726c5f..d3756b8 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -114,7 +114,8 @@ public class DataType implements Serializable {
 public static final DataType ANY = DataType.getType("any");
 
 static {
-MeasureTypeFactory.init();
+//to ensure the MeasureTypeFactory class has initialized
+MeasureTypeFactory.getUDAFs();
 }
 
 public static DataType getType(String type) {

Jenkins build is back to stable : Kylin-Master-JDK-1.7 #186

[Apache Jenkins Server]

See 

kylin git commit: minor, use local metadata to avoid conflict when running concurrently

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/master 8263752a4 -> d135bdb0e


minor, use local metadata to avoid conflict when running concurrently

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d135bdb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d135bdb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d135bdb0

Branch: refs/heads/master
Commit: d135bdb0eed6118245a606fb0bca996420dee70f
Parents: 8263752
Author: etherge 
Authored: Wed Feb 8 18:40:07 2017 +0800
Committer: lidongsjtu 
Committed: Wed Feb 8 18:54:06 2017 +0800

--
 kylin-it/pom.xml  |  3 +++
 .../kylin/job/BaseTestDistributedScheduler.java   | 14 ++
 2 files changed, 13 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d135bdb0/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 91104ba..d58a895 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -36,6 +36,7 @@
 
 
 
+
 
 
 
@@ -316,6 +317,7 @@
 
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
+
-DbuildCubeUsingProvidedData=${buildCubeUsingProvidedData}
 
-DengineType=${engineType}
 
-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 -classpath
@@ -338,6 +340,7 @@
 
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
+
-DbuildCubeUsingProvidedData=${buildCubeUsingProvidedData}
 
-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 -classpath
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d135bdb0/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java 
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 1a0a39d..2f37a50 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
 
 import javax.annotation.Nullable;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
@@ -48,6 +49,7 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
+import com.google.common.io.Files;
 
 public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
 static ExecutableManager execMgr;
@@ -57,13 +59,13 @@ public class BaseTestDistributedScheduler extends 
HBaseMetadataTestCase {
 static KylinConfig kylinConfig1;
 static KylinConfig kylinConfig2;
 static CuratorFramework zkClient;
+static File localMetaDir;
 
 static final String SEGMENT_ID = "segmentId";
 static final String segmentId1 = "segmentId1";
 static final String segmentId2 = "segmentId2";
 static final String serverName1 = "serverName1";
 static final String serverName2 = "serverName2";
-static final String confSrcPath = 
"../examples/test_case_data/sandbox/kylin.properties";
 static final String confDstPath1 = 
"target/kylin_metadata_dist_lock_test1/kylin.properties";
 static final String confDstPath2 = 
"target/kylin_metadata_dist_lock_test2/kylin.properties";
 
@@ -77,14 +79,17 @@ public class BaseTestDistributedScheduler extends 
HBaseMetadataTestCase {
 new File(confDstPath1).getParentFile().mkdirs();
 new File(confDstPath2).getParentFile().mkdirs();
 KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
+
+localMetaDir = Files.createTempDir();
 String backup = srcConfig.getMetadataUrl();
-srcConfig.setProperty("kylin.metadata.url", 
"kylin_metadata_dist_lock_test@hbase");
+srcConfig.setProperty("kylin.metadata.url", 
localMetaDir.getAbsolutePath());
 

Jenkins build became unstable: Kylin-Master-JDK-1.7 #185

[Apache Jenkins Server]

See