[6/7] kylin git commit: KYLIN-2314 Use col identity instead of col name in dictionary refs

[liyang]

KYLIN-2314 Use col identity instead of col name in dictionary refs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6c67197
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6c67197
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6c67197

Branch: refs/heads/KYLIN-2314
Commit: c6c6719730e05c66c14407cfd7926a0a54a6009a
Parents: 30d3763
Author: Li Yang 
Authored: Fri Dec 23 18:06:58 2016 +0800
Committer: Yang Li 
Committed: Sat Dec 24 14:46:39 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  26 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |  30 +-
 .../org/apache/kylin/cube/CubeDescTest.java |  40 +-
 .../kylin/metadata/MetadataManagerTest.java |  67 ++-
 .../kylin/metadata/model/JoinsTreeTest.java |  11 +-
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 -
 .../engine/mr/steps/CreateDictionaryJob.java|   5 +-
 .../engine/mr/steps/CubingExecutableUtil.java   |   8 -
 .../engine/mr/steps/FactDistinctColumnsJob.java |   3 +
 .../mr/steps/FactDistinctColumnsReducer.java|  39 +-
 .../mr/steps/FactDistinctHiveColumnsMapper.java |  81 ++--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   4 +-
 .../mr/steps/UpdateCubeInfoAfterMergeStep.java  |   1 -
 .../localmeta/cube_desc/ci_inner_join_cube.json | 248 ---
 .../localmeta/cube_desc/ci_left_join_cube.json  | 248 ---
 .../test_kylin_snowflake_sales_cube.json| 268 
 .../localmeta/data/DEFAULT.TEST_COUNTRY.csv | 244 +++
 .../model_desc/ci_inner_join_model.json | 100 -
 .../model_desc/ci_left_join_model.json  | 100 -
 .../test_kylin_snowflake_sales_model.json   |  87 
 .../localmeta/table/DEFAULT.TEST_ACCOUNT.json   |  34 ++
 .../localmeta/table/DEFAULT.TEST_COUNTRY.json   |  24 ++
 .../table/DEFAULT.TEST_KYLIN_COUNTRY.json   |  24 --
 .../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json |  28 --
 .../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json  | 408 ---
 .../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json  | 152 ---
 .../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json |  24 --
 .../localmeta/table/SNOWTEST.KYLIN_SALES.json   |  56 ---
 .../java/org/apache/kylin/query/H2Database.java |  11 +-
 30 files changed, 1080 insertions(+), 1295 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 564a4e2..089c3ed 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -125,11 +125,15 @@ public class DeployUtil {
 static final String TABLE_CATEGORY_GROUPINGS = 
"default.test_category_groupings";
 static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
 static final String TABLE_ORDER = "default.test_order";
+static final String TABLE_ACCOUNT = "default.test_account";
+static final String TABLE_COUNTRY = "default.test_country";
 static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
 static final String TABLE_SELLER_TYPE_DIM_TABLE = 
"edw.test_seller_type_dim_table";
 static final String TABLE_SITES = "edw.test_sites";
 
-static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, 
TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, 
TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
+static final String[] TABLE_NAMES = new String[] { //
+TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, 
TABLE_KYLIN_FACT, //
+TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES, TABLE_ACCOUNT, 
TABLE_COUNTRY };
 
 public static void prepareTestDataForNormalCubes(String modelName) throws 
Exception {
 
@@ -228,21 +232,15 @@ public class DeployUtil {
 IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_ORDER.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase(;
-

[7/7] kylin git commit: minor, add time log for BuildCubeWithStream

[liyang]

minor, add time log for BuildCubeWithStream


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4881e282
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4881e282
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4881e282

Branch: refs/heads/KYLIN-2314
Commit: 4881e282eac3d836fb5ab3effca46f8a3808da41
Parents: c6c6719
Author: Yang Li 
Authored: Sat Dec 24 15:34:48 2016 +0800
Committer: Yang Li 
Committed: Sat Dec 24 15:34:48 2016 +0800

--
 .../org/apache/kylin/provision/BuildCubeWithStream.java   | 10 --
 1 file changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4881e282/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 8abb84c..194ea35 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -311,6 +311,9 @@ public class BuildCubeWithStream {
 }
 
 public static void main(String[] args) throws Exception {
+long start = System.currentTimeMillis();
+int exitCode = 0;
+
 BuildCubeWithStream buildCubeWithStream = null;
 try {
 beforeClass();
@@ -320,11 +323,14 @@ public class BuildCubeWithStream {
 logger.info("Build is done");
 buildCubeWithStream.cleanup();
 logger.info("Going to exit");
-System.exit(0);
 } catch (Throwable e) {
 logger.error("error", e);
-System.exit(1);
+exitCode = 1;
 }
 
+long millis = System.currentTimeMillis() - start;
+System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + 
BuildCubeWithStream.class.getName());
+
+System.exit(exitCode);
 }
 }

[2/7] kylin git commit: KYLIN-2287 rm deprecated code & add streaming info when show storage

[liyang]

KYLIN-2287 rm deprecated code & add streaming info when show storage


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/488283d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/488283d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/488283d3

Branch: refs/heads/KYLIN-2314
Commit: 488283d3de1a385a4a6ed03ebd2e412cb9f2f7d7
Parents: 4496928
Author: zhongjian 
Authored: Fri Dec 23 21:35:45 2016 +0800
Committer: zhongjian 
Committed: Fri Dec 23 21:35:45 2016 +0800

--
 webapp/app/js/controllers/cube.js  | 6 ++
 webapp/app/js/controllers/cubeEdit.js  | 3 ---
 webapp/app/js/controllers/sourceMeta.js| 3 ---
 webapp/app/partials/cubes/cube_detail.html | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/cube.js
--
diff --git a/webapp/app/js/controllers/cube.js 
b/webapp/app/js/controllers/cube.js
index 57b53e7..9e675d2 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -88,6 +88,12 @@ KylinApp.controller('CubeCtrl', function ($scope, 
AccessService, MessageService,
 CubeService.getHbaseInfo({cubeId: cube.name, propValue: null, 
action: null}, function (hbase) {
 cube.hbase = hbase;
 
+
TableService.get({tableName:cube.model.fact_table},function(table) {
+  if (table && table.source_type == 1) {
+cube.streaming = true;
+  }
+})
+
 // Calculate cube total size based on each htable.
 var totalSize = 0;
 hbase.forEach(function(t) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 90cdbc0..edbb421 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -311,9 +311,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 "mode": 'edit'
   };
 
-  $scope.cubeState={
-"isStreaming": false
-  }
 
   //fetch cube info and model info in edit model
   // ~ init

http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/sourceMeta.js
--
diff --git a/webapp/app/js/controllers/sourceMeta.js 
b/webapp/app/js/controllers/sourceMeta.js
index dc3c421..bbb9915 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -552,9 +552,6 @@ KylinApp
 
 var StreamingSourceCtrl = function ($scope, 
$location,$interpolate,$templateCache, $modalInstance, tableNames, 
MessageService, projectName, scope, 
tableConfig,cubeConfig,StreamingModel,StreamingService) {
 
-  $scope.cubeState={
-"isStreaming": false
-  }
   $scope.state={
 'mode':'edit'
   }

http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/partials/cubes/cube_detail.html
--
diff --git a/webapp/app/partials/cubes/cube_detail.html 
b/webapp/app/partials/cubes/cube_detail.html
index 1896b42..4319b94 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -44,7 +44,7 @@
 
 
 
+ ng-init="state={mode:'view', cubeName:cube.name};">
 
 
 

[3/7] kylin git commit: KYLIN-2306 fall back to abstract when NoClassFoundException

[liyang]

KYLIN-2306 fall back to abstract when NoClassFoundException


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73a40296
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73a40296
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73a40296

Branch: refs/heads/KYLIN-2314
Commit: 73a40296e33921a000cb97faa3321dd53b8dbf41
Parents: 488283d
Author: Billy Liu 
Authored: Fri Dec 23 23:49:02 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:49:20 2016 +0800

--
 .../kylin/job/execution/ExecutableManager.java  | 59 
 1 file changed, 35 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/73a40296/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0c49a3e..0273fd8 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -44,10 +44,16 @@ public class ExecutableManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(ExecutableManager.class);
 private static final ConcurrentHashMap 
CACHE = new ConcurrentHashMap();
-
+
 private final KylinConfig config;
 private final ExecutableDao executableDao;
 
+private ExecutableManager(KylinConfig config) {
+logger.info("Using metadata url: " + config);
+this.config = config;
+this.executableDao = ExecutableDao.getInstance(config);
+}
+
 public static ExecutableManager getInstance(KylinConfig config) {
 ExecutableManager r = CACHE.get(config);
 if (r == null) {
@@ -69,10 +75,20 @@ public class ExecutableManager {
 CACHE.clear();
 }
 
-private ExecutableManager(KylinConfig config) {
-logger.info("Using metadata url: " + config);
-this.config = config;
-this.executableDao = ExecutableDao.getInstance(config);
+private static ExecutablePO parse(AbstractExecutable executable) {
+ExecutablePO result = new ExecutablePO();
+result.setName(executable.getName());
+result.setUuid(executable.getId());
+result.setType(executable.getClass().getName());
+result.setParams(executable.getParams());
+if (executable instanceof ChainedExecutable) {
+List tasks = Lists.newArrayList();
+for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
+tasks.add(parse(task));
+}
+result.setTasks(tasks);
+}
+return result;
 }
 
 public void addJob(AbstractExecutable executable) {
@@ -321,7 +337,6 @@ public class ExecutableManager {
 updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
 }
 
-
 public void rollbackJob(String jobId, String stepId) {
 AbstractExecutable job = getJob(jobId);
 if (job == null) {
@@ -409,22 +424,6 @@ public class ExecutableManager {
 addJobInfo(id, info);
 }
 
-private static ExecutablePO parse(AbstractExecutable executable) {
-ExecutablePO result = new ExecutablePO();
-result.setName(executable.getName());
-result.setUuid(executable.getId());
-result.setType(executable.getClass().getName());
-result.setParams(executable.getParams());
-if (executable instanceof ChainedExecutable) {
-List tasks = Lists.newArrayList();
-for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
-tasks.add(parse(task));
-}
-result.setTasks(tasks);
-}
-return result;
-}
-
 private AbstractExecutable parseTo(ExecutablePO executablePO) {
 if (executablePO == null) {
 logger.warn("executablePO is null");
@@ -457,8 +456,14 @@ public class ExecutableManager {
 logger.warn("executablePO is null");
 return null;
 }
+String type = executablePO.getType();
 try {
-Class clazz = 
ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
+Class clazz = null;
+try {
+clazz = ClassUtil.forName(type, AbstractExecutable.class);
+} catch (ClassNotFoundException e) {
+clazz = ClassUtil.forName(expectedClass.getName(), 
AbstractExecutable.class);
+}
 Constructor constructor = 

kylin git commit: KYLIN-2318 query cache is not working

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang22 da5e5d693 -> 8596ccf26


KYLIN-2318 query cache is not working


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8596ccf2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8596ccf2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8596ccf2

Branch: refs/heads/yang22
Commit: 8596ccf262241826cb60baf3a9a3999c39ae0c66
Parents: da5e5d6
Author: Hongbin Ma 
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 15:31:22 2016 +0800

--
 .../org/apache/kylin/common/QueryContext.java   | 61 
 .../kylin/common/debug/BackdoorToggles.java | 11 ++--
 .../apache/kylin/rest/request/SQLRequest.java   | 23 +---
 .../apache/kylin/rest/service/QueryService.java | 23 
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  4 +-
 5 files changed, 93 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java 
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for 
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new 
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.Pair;
 
 /**
+ * BackdoorToggles and QueryContext are similar because they're both hosting 
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work 
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ * 
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to 
SQLRequest
  */
 public class BackdoorToggles {
 
@@ -65,10 +70,6 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-}
-
 public static 

kylin git commit: KYLIN-2314 Use col identity instead of col name in dictionary refs

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2314 ab65bd56e -> 8cbd6a6c1


KYLIN-2314 Use col identity instead of col name in dictionary refs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8cbd6a6c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8cbd6a6c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8cbd6a6c

Branch: refs/heads/KYLIN-2314
Commit: 8cbd6a6c14f9639145faa2b8ea728e5d4f510c46
Parents: ab65bd5
Author: Yang Li 
Authored: Sat Dec 24 13:14:37 2016 +0800
Committer: Yang Li 
Committed: Sat Dec 24 13:14:37 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java | 30 
 .../kylin/engine/mr/JobBuilderSupport.java  |  2 -
 .../engine/mr/steps/CreateDictionaryJob.java|  5 +-
 .../engine/mr/steps/CubingExecutableUtil.java   |  8 --
 .../engine/mr/steps/FactDistinctColumnsJob.java |  3 +
 .../mr/steps/FactDistinctColumnsReducer.java| 39 ++
 .../mr/steps/FactDistinctHiveColumnsMapper.java | 81 ++--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  4 +-
 .../mr/steps/UpdateCubeInfoAfterMergeStep.java  |  1 -
 10 files changed, 87 insertions(+), 88 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbd6a6c/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index fe2030a..9670b89 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -240,7 +240,7 @@ public class CubeManager implements IRealizationProvider {
 if (dictInfo != null) {
 Dictionary dict = dictInfo.getDictionaryObject();
 cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
-cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), 
dict.getSize(), dict.getSizeOfId() });
+cubeSeg.getRowkeyStats().add(new Object[] { col.getIdentity(), 
dict.getSize(), dict.getSizeOfId() });
 
 CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
 update.setToUpdateSegs(cubeSeg);

http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbd6a6c/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index e155f86..36a6044 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -98,9 +98,6 @@ public class CubeSegment implements Comparable, 
IBuildable, ISegmen
 @JsonProperty("snapshots")
 private ConcurrentHashMap snapshots; // table name ==> 
snapshot resource path
 
-@JsonProperty("index_path")
-private String indexPath;
-
 @JsonProperty("rowkey_stats")
 private List rowkeyStats = Lists.newArrayList();
 
@@ -296,15 +293,22 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 }
 
 public String getDictResPath(TblColRef col) {
-return getDictionaries().get(dictKey(col));
+String r;
+String dictKey = col.getIdentity();
+r = getDictionaries().get(dictKey);
+
+// try Kylin v1.x dict key as well
+if (r == null) {
+String v1DictKey = col.getTable() + "/" + col.getName();
+r = getDictionaries().get(v1DictKey);
+}
+
+return r;
 }
 
 public void putDictResPath(TblColRef col, String dictResPath) {
-getDictionaries().put(dictKey(col), dictResPath);
-}
-
-private String dictKey(TblColRef col) {
-return col.getTable() + "/" + col.getName();
+String dictKey = col.getIdentity();
+getDictionaries().put(dictKey, dictResPath);
 }
 
 public void setStorageLocationIdentifier(String storageLocationIdentifier) 
{
@@ -523,14 +527,6 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 return cubeInstance;
 }
 
-public String getIndexPath() {
-return indexPath;
-}
-
-public void setIndexPath(String indexPath) {
-this.indexPath = indexPath;
-}
-
 public Map getAdditionalInfo() {
 return additionalInfo;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbd6a6c/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java

kylin git commit: KYLIN-2320 fix getSizeOfId() for empty TrieDictionaryForest

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2314 9ef51f593 -> ab65bd56e


KYLIN-2320 fix getSizeOfId() for empty TrieDictionaryForest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab65bd56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab65bd56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab65bd56

Branch: refs/heads/KYLIN-2314
Commit: ab65bd56edd14fa5fb84b51826a7cd1f8307bb4f
Parents: 9ef51f5
Author: Yang Li 
Authored: Sat Dec 24 07:34:12 2016 +0800
Committer: Yang Li 
Committed: Sat Dec 24 07:34:12 2016 +0800

--
 .../apache/kylin/dict/TrieDictionaryForest.java |  4 +-
 .../kylin/dict/TrieDictionaryForestTest.java| 41 
 2 files changed, 35 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ab65bd56/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 1015b11..e746348 100755
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -87,7 +87,7 @@ public class TrieDictionaryForest extends Dictionary {
 @Override
 public int getSizeOfId() {
 if (trees.isEmpty())
-return -1;
+return 1;
 int maxOffset = accuOffset.get(accuOffset.size() - 1);
 TrieDictionary lastTree = trees.get(trees.size() - 1);
 int sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + 
lastTree.getMaxId() + 1);
@@ -96,7 +96,7 @@ public class TrieDictionaryForest extends Dictionary {
 
 @Override
 public int getSizeOfValue() {
-int maxValue = -1;
+int maxValue = 0;
 for (TrieDictionary tree : trees)
 maxValue = Math.max(maxValue, tree.getSizeOfValue());
 return maxValue;

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab65bd56/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
 
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 32a6ab5..238bb31 100755
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -52,6 +52,14 @@ import org.junit.Test;
 public class TrieDictionaryForestTest {
 
 @Test
+public void testEmptyDict() {
+ArrayList strs = new ArrayList();
+TrieDictionaryForestBuilder builder = newDictBuilder(strs, 0);
+TrieDictionaryForest dict = builder.build();
+assertSameBehaviorAsTrie(dict, strs, 0);
+}
+
+@Test
 public void testBasicFound() {
 ArrayList strs = new ArrayList();
 strs.add("part");
@@ -67,11 +75,11 @@ public class TrieDictionaryForestTest {
 dict.dump(System.out);
 int expectId = baseId;
 for (String s : strs) {
-System.out.println("value:" + s + "  expect id:" + expectId);
 assertEquals(expectId, dict.getIdFromValue(s));
 expectId++;
 }
-System.out.println("test ok");
+
+assertSameBehaviorAsTrie(dict, strs, baseId);
 }
 
 @Test //one string one tree
@@ -95,11 +103,11 @@ public class TrieDictionaryForestTest {
 assertEquals(strs.size(), dict.getTrees().size());
 int expectId = baseId;
 for (String s : strs) {
-System.out.println("value:" + s + "  expect id:" + expectId);
 assertEquals(expectId, dict.getIdFromValue(s));
 expectId++;
 }
-System.out.println("test ok");
+
+assertSameBehaviorAsTrie(dict, strs, baseId);
 }
 
 @Test
@@ -115,11 +123,13 @@ public class TrieDictionaryForestTest {
 dict.dump(System.out);
 //null value query
 int id = dict.getIdFromValue(null, 0);
-System.out.println(id);
+assertEquals(255, id);
 id = dict.getIdFromValue(null, 1);
-System.out.println(id);
+assertEquals(255, id);
 id = dict.getIdFromValue(null, -1);
-System.out.println(id);
+assertEquals(255, id);
+
+assertSameBehaviorAsTrie(dict, strs, 0);
 }
 
 @Test
@@ -940,7 +950,7 @@ public class TrieDictionaryForestTest {
 b.addValue(strs.next());
 return b;
 }

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25a2f9ac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25a2f9ac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25a2f9ac

Branch: refs/heads/yang22-hbase102
Commit: 25a2f9acfdf2fe5d5fc48458b372cc097d75af76
Parents: da5e5d6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:55:51 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-   

[1/4] kylin git commit: KYLIN-2306 fall back to abstract when NoClassFoundException [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 13f41daaf -> c0da9e250 (forced update)


KYLIN-2306 fall back to abstract when NoClassFoundException


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da5e5d69
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da5e5d69
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da5e5d69

Branch: refs/heads/yang22-hbase102
Commit: da5e5d693ac47c9bdf9d11a7bcef373b15076a50
Parents: 7c77a17
Author: Billy Liu 
Authored: Fri Dec 23 23:49:02 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:50:05 2016 +0800

--
 .../kylin/job/execution/ExecutableManager.java  | 59 
 1 file changed, 35 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/da5e5d69/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0c49a3e..0273fd8 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -44,10 +44,16 @@ public class ExecutableManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(ExecutableManager.class);
 private static final ConcurrentHashMap 
CACHE = new ConcurrentHashMap();
-
+
 private final KylinConfig config;
 private final ExecutableDao executableDao;
 
+private ExecutableManager(KylinConfig config) {
+logger.info("Using metadata url: " + config);
+this.config = config;
+this.executableDao = ExecutableDao.getInstance(config);
+}
+
 public static ExecutableManager getInstance(KylinConfig config) {
 ExecutableManager r = CACHE.get(config);
 if (r == null) {
@@ -69,10 +75,20 @@ public class ExecutableManager {
 CACHE.clear();
 }
 
-private ExecutableManager(KylinConfig config) {
-logger.info("Using metadata url: " + config);
-this.config = config;
-this.executableDao = ExecutableDao.getInstance(config);
+private static ExecutablePO parse(AbstractExecutable executable) {
+ExecutablePO result = new ExecutablePO();
+result.setName(executable.getName());
+result.setUuid(executable.getId());
+result.setType(executable.getClass().getName());
+result.setParams(executable.getParams());
+if (executable instanceof ChainedExecutable) {
+List tasks = Lists.newArrayList();
+for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
+tasks.add(parse(task));
+}
+result.setTasks(tasks);
+}
+return result;
 }
 
 public void addJob(AbstractExecutable executable) {
@@ -321,7 +337,6 @@ public class ExecutableManager {
 updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
 }
 
-
 public void rollbackJob(String jobId, String stepId) {
 AbstractExecutable job = getJob(jobId);
 if (job == null) {
@@ -409,22 +424,6 @@ public class ExecutableManager {
 addJobInfo(id, info);
 }
 
-private static ExecutablePO parse(AbstractExecutable executable) {
-ExecutablePO result = new ExecutablePO();
-result.setName(executable.getName());
-result.setUuid(executable.getId());
-result.setType(executable.getClass().getName());
-result.setParams(executable.getParams());
-if (executable instanceof ChainedExecutable) {
-List tasks = Lists.newArrayList();
-for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
-tasks.add(parse(task));
-}
-result.setTasks(tasks);
-}
-return result;
-}
-
 private AbstractExecutable parseTo(ExecutablePO executablePO) {
 if (executablePO == null) {
 logger.warn("executablePO is null");
@@ -457,8 +456,14 @@ public class ExecutableManager {
 logger.warn("executablePO is null");
 return null;
 }
+String type = executablePO.getType();
 try {
-Class clazz = 
ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
+Class clazz = null;
+try {
+clazz = ClassUtil.forName(type, AbstractExecutable.class);
+} catch (ClassNotFoundException e) {
+clazz = 

[4/4] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[billyliu]

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0da9e25
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0da9e25
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0da9e25

Branch: refs/heads/yang22-hbase102
Commit: c0da9e250495d9d4bf72fe74e146f7177f61b281
Parents: 25a2f9a
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:59:17 2016 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c0da9e25/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c0da9e25/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/c0da9e25/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[billyliu]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68fda3f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68fda3f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68fda3f7

Branch: refs/heads/yang22-cdh5.7
Commit: 68fda3f72b5b9edc7f5fc5af9da5ef517887a9f3
Parents: 25a2f9a
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Fri Dec 23 23:57:41 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68fda3f7/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/68fda3f7/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/68fda3f7/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25a2f9ac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25a2f9ac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25a2f9ac

Branch: refs/heads/yang22-cdh5.7
Commit: 25a2f9acfdf2fe5d5fc48458b372cc097d75af76
Parents: da5e5d6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:55:51 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
- 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[1/3] kylin git commit: KYLIN-2306 fall back to abstract when NoClassFoundException [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x afec45c55 -> 25a2f9acf (forced update)


KYLIN-2306 fall back to abstract when NoClassFoundException


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da5e5d69
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da5e5d69
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da5e5d69

Branch: refs/heads/yang22-hbase1.x
Commit: da5e5d693ac47c9bdf9d11a7bcef373b15076a50
Parents: 7c77a17
Author: Billy Liu 
Authored: Fri Dec 23 23:49:02 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:50:05 2016 +0800

--
 .../kylin/job/execution/ExecutableManager.java  | 59 
 1 file changed, 35 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/da5e5d69/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0c49a3e..0273fd8 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -44,10 +44,16 @@ public class ExecutableManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(ExecutableManager.class);
 private static final ConcurrentHashMap 
CACHE = new ConcurrentHashMap();
-
+
 private final KylinConfig config;
 private final ExecutableDao executableDao;
 
+private ExecutableManager(KylinConfig config) {
+logger.info("Using metadata url: " + config);
+this.config = config;
+this.executableDao = ExecutableDao.getInstance(config);
+}
+
 public static ExecutableManager getInstance(KylinConfig config) {
 ExecutableManager r = CACHE.get(config);
 if (r == null) {
@@ -69,10 +75,20 @@ public class ExecutableManager {
 CACHE.clear();
 }
 
-private ExecutableManager(KylinConfig config) {
-logger.info("Using metadata url: " + config);
-this.config = config;
-this.executableDao = ExecutableDao.getInstance(config);
+private static ExecutablePO parse(AbstractExecutable executable) {
+ExecutablePO result = new ExecutablePO();
+result.setName(executable.getName());
+result.setUuid(executable.getId());
+result.setType(executable.getClass().getName());
+result.setParams(executable.getParams());
+if (executable instanceof ChainedExecutable) {
+List tasks = Lists.newArrayList();
+for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
+tasks.add(parse(task));
+}
+result.setTasks(tasks);
+}
+return result;
 }
 
 public void addJob(AbstractExecutable executable) {
@@ -321,7 +337,6 @@ public class ExecutableManager {
 updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
 }
 
-
 public void rollbackJob(String jobId, String stepId) {
 AbstractExecutable job = getJob(jobId);
 if (job == null) {
@@ -409,22 +424,6 @@ public class ExecutableManager {
 addJobInfo(id, info);
 }
 
-private static ExecutablePO parse(AbstractExecutable executable) {
-ExecutablePO result = new ExecutablePO();
-result.setName(executable.getName());
-result.setUuid(executable.getId());
-result.setType(executable.getClass().getName());
-result.setParams(executable.getParams());
-if (executable instanceof ChainedExecutable) {
-List tasks = Lists.newArrayList();
-for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
-tasks.add(parse(task));
-}
-result.setTasks(tasks);
-}
-return result;
-}
-
 private AbstractExecutable parseTo(ExecutablePO executablePO) {
 if (executablePO == null) {
 logger.warn("executablePO is null");
@@ -457,8 +456,14 @@ public class ExecutableManager {
 logger.warn("executablePO is null");
 return null;
 }
+String type = executablePO.getType();
 try {
-Class clazz = 
ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
+Class clazz = null;
+try {
+clazz = ClassUtil.forName(type, AbstractExecutable.class);
+} catch (ClassNotFoundException e) {
+clazz = 

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25a2f9ac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25a2f9ac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25a2f9ac

Branch: refs/heads/yang22-hbase1.x
Commit: 25a2f9acfdf2fe5d5fc48458b372cc097d75af76
Parents: da5e5d6
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:55:51 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/25a2f9ac/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-   

kylin git commit: KYLIN-2306 fall back to abstract when NoClassFoundException

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22 7c77a17f5 -> da5e5d693


KYLIN-2306 fall back to abstract when NoClassFoundException


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da5e5d69
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da5e5d69
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da5e5d69

Branch: refs/heads/yang22
Commit: da5e5d693ac47c9bdf9d11a7bcef373b15076a50
Parents: 7c77a17
Author: Billy Liu 
Authored: Fri Dec 23 23:49:02 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:50:05 2016 +0800

--
 .../kylin/job/execution/ExecutableManager.java  | 59 
 1 file changed, 35 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/da5e5d69/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0c49a3e..0273fd8 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -44,10 +44,16 @@ public class ExecutableManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(ExecutableManager.class);
 private static final ConcurrentHashMap 
CACHE = new ConcurrentHashMap();
-
+
 private final KylinConfig config;
 private final ExecutableDao executableDao;
 
+private ExecutableManager(KylinConfig config) {
+logger.info("Using metadata url: " + config);
+this.config = config;
+this.executableDao = ExecutableDao.getInstance(config);
+}
+
 public static ExecutableManager getInstance(KylinConfig config) {
 ExecutableManager r = CACHE.get(config);
 if (r == null) {
@@ -69,10 +75,20 @@ public class ExecutableManager {
 CACHE.clear();
 }
 
-private ExecutableManager(KylinConfig config) {
-logger.info("Using metadata url: " + config);
-this.config = config;
-this.executableDao = ExecutableDao.getInstance(config);
+private static ExecutablePO parse(AbstractExecutable executable) {
+ExecutablePO result = new ExecutablePO();
+result.setName(executable.getName());
+result.setUuid(executable.getId());
+result.setType(executable.getClass().getName());
+result.setParams(executable.getParams());
+if (executable instanceof ChainedExecutable) {
+List tasks = Lists.newArrayList();
+for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
+tasks.add(parse(task));
+}
+result.setTasks(tasks);
+}
+return result;
 }
 
 public void addJob(AbstractExecutable executable) {
@@ -321,7 +337,6 @@ public class ExecutableManager {
 updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
 }
 
-
 public void rollbackJob(String jobId, String stepId) {
 AbstractExecutable job = getJob(jobId);
 if (job == null) {
@@ -409,22 +424,6 @@ public class ExecutableManager {
 addJobInfo(id, info);
 }
 
-private static ExecutablePO parse(AbstractExecutable executable) {
-ExecutablePO result = new ExecutablePO();
-result.setName(executable.getName());
-result.setUuid(executable.getId());
-result.setType(executable.getClass().getName());
-result.setParams(executable.getParams());
-if (executable instanceof ChainedExecutable) {
-List tasks = Lists.newArrayList();
-for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
-tasks.add(parse(task));
-}
-result.setTasks(tasks);
-}
-return result;
-}
-
 private AbstractExecutable parseTo(ExecutablePO executablePO) {
 if (executablePO == null) {
 logger.warn("executablePO is null");
@@ -457,8 +456,14 @@ public class ExecutableManager {
 logger.warn("executablePO is null");
 return null;
 }
+String type = executablePO.getType();
 try {
-Class clazz = 
ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
+Class clazz = null;
+try {
+clazz = ClassUtil.forName(type, AbstractExecutable.class);
+} catch (ClassNotFoundException e) {
+clazz = ClassUtil.forName(expectedClass.getName(), 

kylin git commit: KYLIN-2306 fall back to abstract when NoClassFoundException

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 488283d3d -> 73a40296e


KYLIN-2306 fall back to abstract when NoClassFoundException


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73a40296
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73a40296
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73a40296

Branch: refs/heads/master
Commit: 73a40296e33921a000cb97faa3321dd53b8dbf41
Parents: 488283d
Author: Billy Liu 
Authored: Fri Dec 23 23:49:02 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 23:49:20 2016 +0800

--
 .../kylin/job/execution/ExecutableManager.java  | 59 
 1 file changed, 35 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/73a40296/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0c49a3e..0273fd8 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -44,10 +44,16 @@ public class ExecutableManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(ExecutableManager.class);
 private static final ConcurrentHashMap 
CACHE = new ConcurrentHashMap();
-
+
 private final KylinConfig config;
 private final ExecutableDao executableDao;
 
+private ExecutableManager(KylinConfig config) {
+logger.info("Using metadata url: " + config);
+this.config = config;
+this.executableDao = ExecutableDao.getInstance(config);
+}
+
 public static ExecutableManager getInstance(KylinConfig config) {
 ExecutableManager r = CACHE.get(config);
 if (r == null) {
@@ -69,10 +75,20 @@ public class ExecutableManager {
 CACHE.clear();
 }
 
-private ExecutableManager(KylinConfig config) {
-logger.info("Using metadata url: " + config);
-this.config = config;
-this.executableDao = ExecutableDao.getInstance(config);
+private static ExecutablePO parse(AbstractExecutable executable) {
+ExecutablePO result = new ExecutablePO();
+result.setName(executable.getName());
+result.setUuid(executable.getId());
+result.setType(executable.getClass().getName());
+result.setParams(executable.getParams());
+if (executable instanceof ChainedExecutable) {
+List tasks = Lists.newArrayList();
+for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
+tasks.add(parse(task));
+}
+result.setTasks(tasks);
+}
+return result;
 }
 
 public void addJob(AbstractExecutable executable) {
@@ -321,7 +337,6 @@ public class ExecutableManager {
 updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
 }
 
-
 public void rollbackJob(String jobId, String stepId) {
 AbstractExecutable job = getJob(jobId);
 if (job == null) {
@@ -409,22 +424,6 @@ public class ExecutableManager {
 addJobInfo(id, info);
 }
 
-private static ExecutablePO parse(AbstractExecutable executable) {
-ExecutablePO result = new ExecutablePO();
-result.setName(executable.getName());
-result.setUuid(executable.getId());
-result.setType(executable.getClass().getName());
-result.setParams(executable.getParams());
-if (executable instanceof ChainedExecutable) {
-List tasks = Lists.newArrayList();
-for (AbstractExecutable task : ((ChainedExecutable) 
executable).getTasks()) {
-tasks.add(parse(task));
-}
-result.setTasks(tasks);
-}
-return result;
-}
-
 private AbstractExecutable parseTo(ExecutablePO executablePO) {
 if (executablePO == null) {
 logger.warn("executablePO is null");
@@ -457,8 +456,14 @@ public class ExecutableManager {
 logger.warn("executablePO is null");
 return null;
 }
+String type = executablePO.getType();
 try {
-Class clazz = 
ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
+Class clazz = null;
+try {
+clazz = ClassUtil.forName(type, AbstractExecutable.class);
+} catch (ClassNotFoundException e) {
+clazz = ClassUtil.forName(expectedClass.getName(), 

[2/2] kylin git commit: KYLIN-2287 rm deprecated code & add streaming info when show storage

[zhongjian]

KYLIN-2287 rm deprecated code & add streaming info when show storage


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/488283d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/488283d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/488283d3

Branch: refs/heads/master
Commit: 488283d3de1a385a4a6ed03ebd2e412cb9f2f7d7
Parents: 4496928
Author: zhongjian 
Authored: Fri Dec 23 21:35:45 2016 +0800
Committer: zhongjian 
Committed: Fri Dec 23 21:35:45 2016 +0800

--
 webapp/app/js/controllers/cube.js  | 6 ++
 webapp/app/js/controllers/cubeEdit.js  | 3 ---
 webapp/app/js/controllers/sourceMeta.js| 3 ---
 webapp/app/partials/cubes/cube_detail.html | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/cube.js
--
diff --git a/webapp/app/js/controllers/cube.js 
b/webapp/app/js/controllers/cube.js
index 57b53e7..9e675d2 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -88,6 +88,12 @@ KylinApp.controller('CubeCtrl', function ($scope, 
AccessService, MessageService,
 CubeService.getHbaseInfo({cubeId: cube.name, propValue: null, 
action: null}, function (hbase) {
 cube.hbase = hbase;
 
+
TableService.get({tableName:cube.model.fact_table},function(table) {
+  if (table && table.source_type == 1) {
+cube.streaming = true;
+  }
+})
+
 // Calculate cube total size based on each htable.
 var totalSize = 0;
 hbase.forEach(function(t) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 90cdbc0..edbb421 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -311,9 +311,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 "mode": 'edit'
   };
 
-  $scope.cubeState={
-"isStreaming": false
-  }
 
   //fetch cube info and model info in edit model
   // ~ init

http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/sourceMeta.js
--
diff --git a/webapp/app/js/controllers/sourceMeta.js 
b/webapp/app/js/controllers/sourceMeta.js
index dc3c421..bbb9915 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -552,9 +552,6 @@ KylinApp
 
 var StreamingSourceCtrl = function ($scope, 
$location,$interpolate,$templateCache, $modalInstance, tableNames, 
MessageService, projectName, scope, 
tableConfig,cubeConfig,StreamingModel,StreamingService) {
 
-  $scope.cubeState={
-"isStreaming": false
-  }
   $scope.state={
 'mode':'edit'
   }

http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/partials/cubes/cube_detail.html
--
diff --git a/webapp/app/partials/cubes/cube_detail.html 
b/webapp/app/partials/cubes/cube_detail.html
index 1896b42..4319b94 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -44,7 +44,7 @@
 
 
 
+ ng-init="state={mode:'view', cubeName:cube.name};">
 
 
 

[1/2] kylin git commit: minor, rm deprecated code

[zhongjian]

Repository: kylin
Updated Branches:
  refs/heads/master 120df2d55 -> 488283d3d


minor, rm deprecated code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4496928e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4496928e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4496928e

Branch: refs/heads/master
Commit: 4496928e31abd002636d7695f0bf118210dae6eb
Parents: 120df2d
Author: zhongjian 
Authored: Fri Dec 23 20:48:27 2016 +0800
Committer: zhongjian 
Committed: Fri Dec 23 20:48:27 2016 +0800

--
 webapp/app/partials/cubes/cubes.html | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4496928e/webapp/app/partials/cubes/cubes.html
--
diff --git a/webapp/app/partials/cubes/cubes.html 
b/webapp/app/partials/cubes/cubes.html
index 4be7ecd..7fc3adb 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -92,7 +92,6 @@
 Drop
 
 Edit
-
 Build
 Refresh
 Merge

kylin git commit: fix UT

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2314 523e6a7ff -> 9ef51f593


fix UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ef51f59
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ef51f59
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ef51f59

Branch: refs/heads/KYLIN-2314
Commit: 9ef51f59388e2bf172878028dfc83f189e71e963
Parents: 523e6a7
Author: Yang Li 
Authored: Fri Dec 23 21:04:35 2016 +0800
Committer: Yang Li 
Committed: Fri Dec 23 21:04:35 2016 +0800

--
 .../localmeta/cube_desc/ci_inner_join_cube.json |  34 +--
 .../localmeta/cube_desc/ci_left_join_cube.json  |  34 +--
 .../test_kylin_snowflake_sales_cube.json| 268 ---
 3 files changed, 34 insertions(+), 302 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ef51f59/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json 
b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 204e09a..1ebd7f3 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -62,11 +62,11 @@
   }, {
 "name" : "SELLER_BUYER_LEVEL",
 "table" : "SELLER_ACCOUNT",
-"column" : "BUYER_LEVEL"
+"column" : "ACCOUNT_BUYER_LEVEL"
   }, {
 "name" : "SELLER_SELLER_LEVEL",
 "table" : "SELLER_ACCOUNT",
-"column" : "SELLER_LEVEL"
+"column" : "ACCOUNT_SELLER_LEVEL"
   }, {
 "name" : "SELLER_COUNTRY",
 "table" : "SELLER_ACCOUNT",
@@ -82,11 +82,11 @@
   }, {
 "name" : "BUYER_BUYER_LEVEL",
 "table" : "BUYER_ACCOUNT",
-"column" : "BUYER_LEVEL"
+"column" : "ACCOUNT_BUYER_LEVEL"
   }, {
 "name" : "BUYER_SELLER_LEVEL",
 "table" : "BUYER_ACCOUNT",
-"column" : "SELLER_LEVEL"
+"column" : "ACCOUNT_SELLER_LEVEL"
   }, {
 "name" : "BUYER_COUNTRY",
 "table" : "BUYER_ACCOUNT",
@@ -201,7 +201,7 @@
   "expression" : "EXTENDED_COLUMN",
   "parameter": {
 "type": "column",
-"value": "TEST_FACT_KYLIN.ORDER_ID",
+"value": "TEST_KYLIN_FACT.ORDER_ID",
 "next_parameter": {
   "type": "column",
   "value": "TEST_ORDER.TEST_EXTENDED_COLUMN"
@@ -313,10 +313,10 @@
   "column" : "TEST_ORDER.BUYER_ID",
   "encoding" : "int:4"
 }, {
-  "column" : "BUYER_ACCOUNT.BUYER_LEVEL",
+  "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
   "encoding" : "dict"
 }, {
-  "column" : "BUYER_ACCOUNT.SELLER_LEVEL",
+  "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
   "encoding" : "dict"
 }, {
   "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
@@ -325,10 +325,10 @@
   "column" : "BUYER_COUNTRY.NAME",
   "encoding" : "dict"
 }, {
-  "column" : "SELLER_ACCOUNT.BUYER_LEVEL",
+  "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
   "encoding" : "dict"
 }, {
-  "column" : "SELLER_ACCOUNT.SELLER_LEVEL",
+  "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
   "encoding" : "dict"
 }, {
   "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
@@ -392,8 +392,8 @@
"TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 

"TEST_KYLIN_FACT.SELLER_ID", 
-   "SELLER_ACCOUNT.BUYER_LEVEL",
-   "SELLER_ACCOUNT.SELLER_LEVEL",
+   "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+   "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
"SELLER_ACCOUNT.ACCOUNT_COUNTRY",
"SELLER_COUNTRY.NAME",

@@ -401,8 +401,8 @@
"TEST_ORDER.TEST_DATE_ENC", 
"TEST_ORDER.TEST_TIME_ENC",
"TEST_ORDER.BUYER_ID",
-   "BUYER_ACCOUNT.BUYER_LEVEL",
-   "BUYER_ACCOUNT.SELLER_LEVEL",
+   "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+   "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
"BUYER_ACCOUNT.ACCOUNT_COUNTRY",
"BUYER_COUNTRY.NAME" ],
 "select_rule" : {
@@ -419,8 +419,8 @@
  "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ],

[ "TEST_KYLIN_FACT.SELLER_ID", 
- "SELLER_ACCOUNT.BUYER_LEVEL",
- "SELLER_ACCOUNT.SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
  "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
  "SELLER_COUNTRY.NAME" ],

kylin git commit: halfway, pending build new CI cube locally

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2314 [created] 523e6a7ff


halfway, pending build new CI cube locally


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/523e6a7f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/523e6a7f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/523e6a7f

Branch: refs/heads/KYLIN-2314
Commit: 523e6a7ff72358e91ae73afb447efe26767d6469
Parents: 120df2d
Author: Li Yang 
Authored: Fri Dec 23 18:06:58 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 23 18:06:58 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  26 +-
 .../org/apache/kylin/cube/CubeDescTest.java |  40 +-
 .../kylin/metadata/MetadataManagerTest.java |  67 ++-
 .../kylin/metadata/model/JoinsTreeTest.java |  11 +-
 .../localmeta/cube_desc/ci_inner_join_cube.json | 248 ---
 .../localmeta/cube_desc/ci_left_join_cube.json  | 248 ---
 .../localmeta/data/DEFAULT.TEST_COUNTRY.csv | 244 +++
 .../model_desc/ci_inner_join_model.json | 100 -
 .../model_desc/ci_left_join_model.json  | 100 -
 .../test_kylin_snowflake_sales_model.json   |  87 
 .../localmeta/table/DEFAULT.TEST_ACCOUNT.json   |  34 ++
 .../localmeta/table/DEFAULT.TEST_COUNTRY.json   |  24 ++
 .../table/DEFAULT.TEST_KYLIN_COUNTRY.json   |  24 --
 .../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json |  28 --
 .../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json  | 408 ---
 .../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json  | 152 ---
 .../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json |  24 --
 .../localmeta/table/SNOWTEST.KYLIN_SALES.json   |  56 ---
 .../java/org/apache/kylin/query/H2Database.java |  11 +-
 19 files changed, 993 insertions(+), 939 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/523e6a7f/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 564a4e2..089c3ed 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -125,11 +125,15 @@ public class DeployUtil {
 static final String TABLE_CATEGORY_GROUPINGS = 
"default.test_category_groupings";
 static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
 static final String TABLE_ORDER = "default.test_order";
+static final String TABLE_ACCOUNT = "default.test_account";
+static final String TABLE_COUNTRY = "default.test_country";
 static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
 static final String TABLE_SELLER_TYPE_DIM_TABLE = 
"edw.test_seller_type_dim_table";
 static final String TABLE_SITES = "edw.test_sites";
 
-static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, 
TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, 
TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
+static final String[] TABLE_NAMES = new String[] { //
+TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, 
TABLE_KYLIN_FACT, //
+TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES, TABLE_ACCOUNT, 
TABLE_COUNTRY };
 
 public static void prepareTestDataForNormalCubes(String modelName) throws 
Exception {
 
@@ -228,21 +232,15 @@ public class DeployUtil {
 IHiveClient hiveClient = HiveClientFactory.getHiveClient();
 // create hive tables
 hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_ORDER.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase(;
+for (String tablename : TABLE_NAMES) {
+
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(tablename.toUpperCase(;
+}
 
 // load data to hive tables
 // LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename
-hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir));
-hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, 
tableFileDir));
-