[08/10] kylin git commit: KYLIN-919 fix ui show dimension error

2015-12-04 Thread mahongbin 
KYLIN-919 fix ui show dimension error

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24d3825b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24d3825b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24d3825b

Branch: refs/heads/2.x-staging
Commit: 24d3825bc4652f2f449dde68223788aa3b51fefe
Parents: 5216fad
Author: jian 
Authored: Fri Dec 4 12:29:09 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/24d3825b/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 2920458..ae4a0f1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -47,6 +47,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return temp;
   };
 
+  //get columns from model
   $scope.getDimColumnsByTable = function (tableName) {
 if (!tableName) {
   return [];
@@ -55,6 +56,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 var tableDim = _.find($scope.metaModel.model.dimensions, function 
(dimension) {
   return dimension.table == tableName
 });
+if(!tableDim){
+  return [];
+}
 var tableDimColumns = tableDim.columns;
 var avaColObject = _.filter(tableColumns, function (col) {
   return tableDimColumns.indexOf(col.name) != -1;

[02/10] kylin git commit: KYLIN-1179 Cannot use String as partition column

2015-12-04 Thread mahongbin 
KYLIN-1179 Cannot use String as partition column

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/225a7d5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/225a7d5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/225a7d5b

Branch: refs/heads/2.x-staging
Commit: 225a7d5b6c68e4c0d5a70283823e4fede59d5522
Parents: 9f60f47
Author: lidongsjtu 
Authored: Mon Nov 30 10:48:36 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:06 2015 +0800

--
 .../hbase/cube/v2/CubeSegmentScanner.java   | 26 +++-
 webapp/app/js/controllers/modelEdit.js  |  2 +-
 2 files changed, 16 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/225a7d5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 1eed318..2f78a86 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -11,6 +11,7 @@ import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.DateFormat;
@@ -30,6 +31,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.GTUtil;
 import org.apache.kylin.gridtable.IGTScanner;
 import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.DataType;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -83,27 +85,23 @@ public class CubeSegmentScanner implements IGTScanner {
 GTInfo trimmedInfo = GTInfo.deserialize(trimmedInfoBytes);
 
 for (GTScanRange range : scanRanges) {
-scanRequests.add(new GTScanRequest(trimmedInfo, 
range.replaceGTInfo(trimmedInfo),
-gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, 
gtFilter, allowPreAggregate));
+scanRequests.add(new GTScanRequest(trimmedInfo, 
range.replaceGTInfo(trimmedInfo), gtDimensions, gtAggrGroups, gtAggrMetrics, 
gtAggrFuncs, gtFilter, allowPreAggregate));
 }
 
 scanner = new Scanner();
 }
 
 private Pair getSegmentStartAndEnd(TblColRef 
tblColRef, int index) {
-
-String partitionColType = tblColRef.getColumnDesc().getDatatype();
-
 ByteArray start;
 if (cubeSeg.getDateRangeStart() != Long.MIN_VALUE) {
-start = translateTsToString(cubeSeg.getDateRangeStart(), 
partitionColType, index);
+start = translateTsToString(cubeSeg.getDateRangeStart(), index, 1);
 } else {
 start = new ByteArray();
 }
 
 ByteArray end;
 if (cubeSeg.getDateRangeEnd() != Long.MAX_VALUE) {
-end = translateTsToString(cubeSeg.getDateRangeEnd(), 
partitionColType, index);
+end = translateTsToString(cubeSeg.getDateRangeEnd(), index, -1);
 } else {
 end = new ByteArray();
 }
@@ -111,19 +109,25 @@ public class CubeSegmentScanner implements IGTScanner {
 
 }
 
-private ByteArray translateTsToString(long ts, String partitionColType, 
int index) {
+private ByteArray translateTsToString(long ts, int index, int 
roundingFlag) {
 String value;
-if ("date".equalsIgnoreCase(partitionColType)) {
+DataType partitionColType = info.getColumnType(index);
+if (partitionColType.isDate()) {
 value = DateFormat.formatToDateStr(ts);
-} else if ("timestamp".equalsIgnoreCase(partitionColType)) {
+} else if (partitionColType.isDatetime()) {
 //TODO: if partition col is not dict encoded, value's format may 
differ from expected. Though by default it is not the case
 value = DateFormat.formatToTimeWithoutMilliStr(ts);
+} else if (partitionColType.isStringFamily()) {
+String partitionDateFormat = 
cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat();
+if (StringUtils.isEmpty(partitionDateFormat))
+partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
+value = DateFormat.formatToDateStr(ts, partitionDateFormat);
 } else {
 throw new RuntimeException("Type "

[10/10] kylin git commit: KYLIN-919 model name validate when edit model, add lowercase check

2015-12-04 Thread mahongbin 
KYLIN-919 model name validate when edit model,add lowercase check

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4afd144a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4afd144a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4afd144a

Branch: refs/heads/2.x-staging
Commit: 4afd144af38f99c2fc7333b18f07f2880cd80434
Parents: 24d3825
Author: Zhong 
Authored: Wed Dec 2 15:02:03 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 webapp/app/js/controllers/modelSchema.js | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4afd144a/webapp/app/js/controllers/modelSchema.js
--
diff --git a/webapp/app/js/controllers/modelSchema.js 
b/webapp/app/js/controllers/modelSchema.js
index 9e9f09c..682ce5e 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -166,9 +166,11 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, 
QueryService, UserServi
 
 var modelName = $scope.modelsManager.selectedModel.name.toUpperCase();
 var models = $scope.modelsManager.modelNameList;
-if ($scope.modelMode=="addNewModel"&(modelName) != -1 || 
models.indexOf(modelName.toLowerCase()) !=-1) {
-  SweetAlert.swal('', "Model named [" + modelName + "] already exist!", 
'warning');
-  return false;
+if ($scope.modelMode=="addNewModel") {
+  if(models.indexOf(modelName) != -1 || 
models.indexOf(modelName.toLowerCase()) !=-1){
+SweetAlert.swal('', "Model named [" + modelName + "] already exist!", 
'warning');
+return false;
+  }
 }
 return true;
   }

kylin git commit: KYLIN-1168 bug fix

2015-12-04 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 62012ad56 -> 95bc7dfa6


KYLIN-1168 bug fix

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95bc7dfa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95bc7dfa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95bc7dfa

Branch: refs/heads/1.x-staging
Commit: 95bc7dfa6d892ac01000f45152f45979b5150c7f
Parents: 62012ad
Author: shaofengshi 
Authored: Fri Dec 4 17:07:16 2015 +0800
Committer: shaofengshi 
Committed: Fri Dec 4 17:07:16 2015 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/CacheService.java  | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/95bc7dfa/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 398577d..38d7400 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -90,8 +90,6 @@ public class CacheService extends BasicService {
 // new data model
 getMetadataManager().reloadDataModelDesc(cacheKey);
 }
-IIDescManager.clearCache();
-CubeDescManager.clearCache();
 break;
 case ALL:
 MetadataManager.clearCache();

kylin git commit: KYLIN-930 list cube realization under each project

2015-12-04 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging eb692f38e -> 62012ad56


KYLIN-930 list cube realization under each project

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62012ad5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62012ad5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62012ad5

Branch: refs/heads/1.x-staging
Commit: 62012ad564b53c2fa666444977fd21cf1f042f00
Parents: eb692f3
Author: jian 
Authored: Fri Dec 4 14:16:09 2015 +0800
Committer: shaofengshi 
Committed: Fri Dec 4 16:15:42 2015 +0800

--
 webapp/app/partials/projects/project_detail.html | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/62012ad5/webapp/app/partials/projects/project_detail.html
--
diff --git a/webapp/app/partials/projects/project_detail.html 
b/webapp/app/partials/projects/project_detail.html
index 35c8ef0..2644f08 100644
--- a/webapp/app/partials/projects/project_detail.html
+++ b/webapp/app/partials/projects/project_detail.html
@@ -32,7 +32,7 @@
 
 
 
-
+
 
 
 Cube Name
@@ -40,14 +40,14 @@
 
 
 
-
-{{cube}}
-Detail
+
+{{item.realization}}
+Detail
 
 
 
 
-
+

[07/10] kylin git commit: minor, update hyperlinks for kylin contacts on webapp

2015-12-04 Thread mahongbin 
minor, update hyperlinks for kylin contacts on webapp

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d10c20dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d10c20dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d10c20dd

Branch: refs/heads/2.x-staging
Commit: d10c20ddea345e3acae21c8d0f2d372f419153ff
Parents: e8e2d27
Author: lidongsjtu 
Authored: Thu Dec 3 11:38:11 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 webapp/app/partials/footer.html | 4 ++--
 webapp/app/partials/login.html  | 2 +-
 webapp/app/partials/models/models_tree.html | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d10c20dd/webapp/app/partials/footer.html
--
diff --git a/webapp/app/partials/footer.html b/webapp/app/partials/footer.html
index 5a72a52..d0a513a 100644
--- a/webapp/app/partials/footer.html
+++ b/webapp/app/partials/footer.html
@@ -20,8 +20,8 @@
 
 
 
-http://kylin.io; style="color:#808080;"> Home Page |
-https://groups.google.com/forum/#!forum/kylin-olap; 
style="color:#808080;"> Google Group
+http://kylin.apache.org; style="color:#808080;"> Apache Kylin |
+http://kylin.apache.org/community/; 
style="color:#808080;"> Apache Kylin Community
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d10c20dd/webapp/app/partials/login.html
--
diff --git a/webapp/app/partials/login.html b/webapp/app/partials/login.html
index e64f9d5..79128b1 100644
--- a/webapp/app/partials/login.html
+++ b/webapp/app/partials/login.html
@@ -56,7 +56,7 @@
 
 
 
-https://groups.google.com/forum/#!forum/kylin-olap; target="_blank" 
class="text-muted">Login Issue?
+http://kylin.apache.org/community/; target="_blank" 
class="text-muted">Login Issue?
 
 
 {{loading? 
'Logging In...': 'Log In'}}

http://git-wip-us.apache.org/repos/asf/kylin/blob/d10c20dd/webapp/app/partials/models/models_tree.html
--
diff --git a/webapp/app/partials/models/models_tree.html 
b/webapp/app/partials/models/models_tree.html
index 30706cf..7b94841 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -26,7 +26,7 @@
 
 
   
- New 
Model
+New 
Model
   
   
 New Cube

[04/10] kylin git commit: KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0

2015-12-04 Thread mahongbin 
KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5216fadd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5216fadd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5216fadd

Branch: refs/heads/2.x-staging
Commit: 5216faddd68df289be590f2f214b84a41011694f
Parents: d10c20d
Author: lidongsjtu 
Authored: Thu Dec 3 11:44:59 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 .../job/upgrade/CubeDescSignatureUpdate.java|  66 ---
 .../job/upgrade/v2/DataModelDescUpgradeV2.java  | 170 +++
 .../kylin/metadata/model/DataModelDesc.java |   8 +
 3 files changed, 217 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5216fadd/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
 
b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
index 178f1b0..36784de 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
@@ -18,57 +18,40 @@
 
 package org.apache.kylin.job.upgrade;
 
-import com.google.common.collect.Lists;
+import java.util.List;
+
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
 
-import java.util.List;
+import com.google.common.collect.Lists;
 
 /**
  * Created by dongli on 11/17/15.
  */
 public class CubeDescSignatureUpdate {
+private static final Log logger = 
LogFactory.getLog(CubeDescSignatureUpdate.class);
 private KylinConfig config = null;
 private ResourceStore store;
 private String[] cubeNames;
 private List updatedResources = Lists.newArrayList();
 private List errorMsgs = Lists.newArrayList();
 
-private static final Log logger = 
LogFactory.getLog(CubeDescSignatureUpdate.class);
-
 public CubeDescSignatureUpdate(String[] cubes) {
 config = KylinConfig.getInstanceFromEnv();
 store = ResourceStore.getStore(config);
 cubeNames = cubes;
 }
 
-public void update() {
-logger.info("Reloading Cube Metadata from store: " + 
store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
-CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
-List cubeDescs;
-if (ArrayUtils.isEmpty(cubeNames)) {
-cubeDescs = cubeDescManager.listAllDesc();
-} else {
-String[] names = cubeNames[0].split(",");
-if (ArrayUtils.isEmpty(names))
-return;
-cubeDescs = Lists.newArrayListWithCapacity(names.length);
-for (String name : names) {
-cubeDescs.add(cubeDescManager.getCubeDesc(name));
-}
-}
-for (CubeDesc cubeDesc : cubeDescs) {
-updateCubeDesc(cubeDesc);
-}
-}
-
 public static void main(String args[]) {
-if (args != null && args.length != 0 && args.length != 1) {
+if (args != null && args.length > 1) {
 System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; 
e.g, cube1,cube2 ");
 return;
 }
@@ -100,11 +83,40 @@ public class CubeDescSignatureUpdate {
 
logger.info("=");
 }
 
+public void update() {
+logger.info("Reloading Cube Metadata from store: " + 
store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+List cubeDescs;
+if (ArrayUtils.isEmpty(cubeNames)) {
+cubeDescs = cubeDescManager.listAllDesc();
+} else {
+String[] names = cubeNames[0].split(",");
+if (ArrayUtils.isEmpty(names))
+return;
+cubeDescs = Lists.newArrayListWithCapacity(names.length);
+for (String name : names) {
+cubeDescs.add(cubeDescManager.getCubeDesc(name));
+}
+}
+for (CubeDesc cubeDesc : cubeDescs) {
+

[05/10] kylin git commit: KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery

2015-12-04 Thread mahongbin 
KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2fb097bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2fb097bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2fb097bb

Branch: refs/heads/2.0-rc
Commit: 2fb097bba79414b70b8a025b893a9ebf59a11258
Parents: fde4a3d
Author: lidongsjtu 
Authored: Thu Dec 3 09:49:38 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:14:34 2015 +0800

--
 .../apache/kylin/query/relnode/OLAPLimitRel.java |  7 +--
 .../resources/query/sql_verifyCount/query04.sql  | 19 +++
 .../query/sql_verifyCount/query04.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query05.sql  | 19 +++
 .../query/sql_verifyCount/query05.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query06.sql  | 19 +++
 .../query/sql_verifyCount/query06.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query07.sql  | 19 +++
 .../query/sql_verifyCount/query07.sql.disable| 19 ---
 9 files changed, 81 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 74d5de0..572a5c7 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -102,8 +102,11 @@ public class OLAPLimitRel extends SingleRel implements 
OLAPRel {
 
 @Override
 public EnumerableRel implementEnumerable(List inputs) {
-return new EnumerableLimit(getCluster(), 
getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
-sole(inputs), localOffset, localFetch);
+EnumerableRel input = sole(inputs);
+if (input instanceof OLAPRel) {
+((OLAPRel) input).replaceTraitSet(EnumerableConvention.INSTANCE);
+}
+return EnumerableLimit.create(input, localOffset, localFetch);
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query04.sql
--
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql 
b/query/src/test/resources/query/sql_verifyCount/query04.sql
new file mode 100644
index 000..9d3e409
--- /dev/null
+++ b/query/src/test/resources/query/sql_verifyCount/query04.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select * from test_kylin_fact limit 100

http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
--
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable 
b/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
deleted file mode 100644
index 9d3e409..000
--- a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,

[08/10] kylin git commit: KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0

2015-12-04 Thread mahongbin 
KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ecbfe72
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ecbfe72
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ecbfe72

Branch: refs/heads/2.0-rc
Commit: 3ecbfe72d94547d9cd3d4c90c98024c8fc915ce5
Parents: 1b6f1fe
Author: lidongsjtu 
Authored: Thu Dec 3 11:44:59 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:14:52 2015 +0800

--
 .../job/upgrade/CubeDescSignatureUpdate.java|  66 ---
 .../job/upgrade/v2/DataModelDescUpgradeV2.java  | 170 +++
 .../kylin/metadata/model/DataModelDesc.java |   8 +
 3 files changed, 217 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ecbfe72/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
 
b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
index 178f1b0..36784de 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
@@ -18,57 +18,40 @@
 
 package org.apache.kylin.job.upgrade;
 
-import com.google.common.collect.Lists;
+import java.util.List;
+
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
 
-import java.util.List;
+import com.google.common.collect.Lists;
 
 /**
  * Created by dongli on 11/17/15.
  */
 public class CubeDescSignatureUpdate {
+private static final Log logger = 
LogFactory.getLog(CubeDescSignatureUpdate.class);
 private KylinConfig config = null;
 private ResourceStore store;
 private String[] cubeNames;
 private List updatedResources = Lists.newArrayList();
 private List errorMsgs = Lists.newArrayList();
 
-private static final Log logger = 
LogFactory.getLog(CubeDescSignatureUpdate.class);
-
 public CubeDescSignatureUpdate(String[] cubes) {
 config = KylinConfig.getInstanceFromEnv();
 store = ResourceStore.getStore(config);
 cubeNames = cubes;
 }
 
-public void update() {
-logger.info("Reloading Cube Metadata from store: " + 
store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
-CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
-List cubeDescs;
-if (ArrayUtils.isEmpty(cubeNames)) {
-cubeDescs = cubeDescManager.listAllDesc();
-} else {
-String[] names = cubeNames[0].split(",");
-if (ArrayUtils.isEmpty(names))
-return;
-cubeDescs = Lists.newArrayListWithCapacity(names.length);
-for (String name : names) {
-cubeDescs.add(cubeDescManager.getCubeDesc(name));
-}
-}
-for (CubeDesc cubeDesc : cubeDescs) {
-updateCubeDesc(cubeDesc);
-}
-}
-
 public static void main(String args[]) {
-if (args != null && args.length != 0 && args.length != 1) {
+if (args != null && args.length > 1) {
 System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; 
e.g, cube1,cube2 ");
 return;
 }
@@ -100,11 +83,40 @@ public class CubeDescSignatureUpdate {
 
logger.info("=");
 }
 
+public void update() {
+logger.info("Reloading Cube Metadata from store: " + 
store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+List cubeDescs;
+if (ArrayUtils.isEmpty(cubeNames)) {
+cubeDescs = cubeDescManager.listAllDesc();
+} else {
+String[] names = cubeNames[0].split(",");
+if (ArrayUtils.isEmpty(names))
+return;
+cubeDescs = Lists.newArrayListWithCapacity(names.length);
+for (String name : names) {
+cubeDescs.add(cubeDescManager.getCubeDesc(name));
+}
+}
+for (CubeDesc cubeDesc : cubeDescs) {
+

[09/10] kylin git commit: KYLIN-1190 Make memory budget per query configurable

2015-12-04 Thread mahongbin 
KYLIN-1190 Make memory budget per query configurable

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e8e2d27a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e8e2d27a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e8e2d27a

Branch: refs/heads/2.x-staging
Commit: e8e2d27ab82d6565615a27d4a441f019a5b5aafa
Parents: 6b20569
Author: lidongsjtu 
Authored: Thu Dec 3 09:50:19 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfig.java  |  4 
 .../kylin/storage/hbase/cube/v1/CubeStorageQuery.java   | 12 
 .../kylin/storage/hbase/cube/v2/CubeStorageQuery.java   | 11 +++
 3 files changed, 19 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index b08df5f..d2a94c5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -492,6 +492,10 @@ public class KylinConfig implements Serializable {
 return 
Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", 
String.valueOf(10 * 1024)));
 }
 
+public long getQueryMemBudget() {
+return Long.parseLong(this.getOptional("kylin.query.mem.budget", 
String.valueOf(3L * 1024 * 1024 * 1024)));
+}
+
 public boolean isQuerySecureEnabled() {
 return 
Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 4d34943..2fa0490 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -82,7 +82,6 @@ public class CubeStorageQuery implements 
ICachableStorageQuery {
 private static final Logger logger = 
LoggerFactory.getLogger(CubeStorageQuery.class);
 
 private static final int MERGE_KEYRANGE_THRESHOLD = 100;
-private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; 
// 3G
 
 private final CubeInstance cubeInstance;
 private final CubeDesc cubeDesc;
@@ -732,8 +731,13 @@ public class CubeStorageQuery implements 
ICachableStorageQuery {
 }
 }
 
-long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
-context.setThreshold((int) rowEst);
+long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / 
rowSizeEst;
+if (rowEst > 0) {
+logger.info("Memory budget is set to: " + rowEst);
+context.setThreshold((int) rowEst);
+} else {
+logger.info("Memory budget is not set.");
+}
 }
 
 private void setLimit(TupleFilter filter, StorageContext context) {
@@ -760,7 +764,7 @@ public class CubeStorageQuery implements 
ICachableStorageQuery {
 topnLiteralCol = func.getTopNLiteralColumn();
 }
 }
-
+
 // if TopN is not involved
 if (topnFunc == null)
 return;

http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 0c8c3bd..59ed99b 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -41,8 +41,6 @@ public class CubeStorageQuery implements 
ICachableStorageQuery {
 
 private static final Logger logger = 
LoggerFactory.getLogger(CubeStorageQuery.class);
 
-private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; 
// 3G
-
 private final CubeInstance cubeInstance;
 private final CubeDesc cubeDesc;
 
@@ -362,8

kylin git commit: KYLIN-919 validate model dimension rule update

2015-12-04 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 83288ac56 -> 519b43aa9


KYLIN-919 validate model dimension rule update


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/519b43aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/519b43aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/519b43aa

Branch: refs/heads/2.0-rc
Commit: 519b43aa9fecfd7a56aae4d97a789c880519c343
Parents: 83288ac
Author: jian 
Authored: Fri Dec 4 18:33:55 2015 +0800
Committer: jian 
Committed: Fri Dec 4 18:33:55 2015 +0800

--
 webapp/app/js/controllers/modelSchema.js | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/519b43aa/webapp/app/js/controllers/modelSchema.js
--
diff --git a/webapp/app/js/controllers/modelSchema.js 
b/webapp/app/js/controllers/modelSchema.js
index 682ce5e..198b91a 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -206,11 +206,15 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, 
QueryService, UserServi
 if (!modelsManager.selectedModel.dimensions.length) {
   errors.push("No dimensions defined.");
 }
+var isDimensionDefined = false;
 angular.forEach(modelsManager.selectedModel.dimensions, function 
(_dimension) {
-  if (!_dimension.columns || !_dimension.columns.length) {
-errors.push("No dimension columns defined for Table[" + 
_dimension.table + "]");
+  if(_dimension.columns && _dimension.columns.length){
+isDimensionDefined = true;
   }
 });
+if(!isDimensionDefined){
+  errors.push("No dimensions defined.");
+}
 var errorInfo = "";
 angular.forEach(errors, function (item) {
   errorInfo += "\n" + item;

[1/5] kylin git commit: KYLIN-976 Extract fillTuple() methods

2015-12-04 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-976 63f376a5a -> 21f56da50


http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
index c395fd9..bde8dd2 100644
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
+++ 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
@@ -26,7 +26,7 @@ import java.util.List;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilterSerializer;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.Tuple;
 import org.junit.Test;
 
 /**

[6/7] kylin git commit: KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming joint semantic

2015-12-04 Thread mahongbin 
KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming 
joint semantic


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07537a9a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07537a9a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07537a9a

Branch: refs/heads/2.x-staging
Commit: 07537a9aa6c6d6ed37fb039bee5b3c2eabd534d8
Parents: af6b11a
Author: honma 
Authored: Fri Nov 27 16:47:28 2015 +0800
Committer: honma 
Committed: Fri Dec 4 17:08:01 2015 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   |   7 +-
 .../persistence/RootPersistentEntity.java   |   2 +
 .../org/apache/kylin/common/util/BasicTest.java |   2 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   6 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java| 329 +++-
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java |  81 +-
 .../kylin/cube/cuboid/CuboidScheduler.java  | 256 +++---
 .../gridtable/CuboidToGridTableMapping.java |  10 +-
 .../cube/inmemcubing/InMemCubeBuilderUtils.java |   6 +-
 .../org/apache/kylin/cube/kv/RowConstants.java  |   2 +
 .../kylin/cube/model/AggregationGroup.java  | 280 +++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 187 ++---
 .../apache/kylin/cube/model/DimensionDesc.java  | 121 +--
 .../kylin/cube/model/HBaseMappingDesc.java  |   2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  73 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java | 229 +
 .../org/apache/kylin/cube/model/SelectRule.java |  30 +
 .../kylin/cube/model/v2/CubeBuildTypeEnum.java  |  39 +
 .../apache/kylin/cube/model/v2/CubeDesc.java| 826 +++
 .../kylin/cube/model/v2/DimensionDesc.java  | 239 ++
 .../kylin/cube/model/v2/HBaseColumnDesc.java| 138 
 .../cube/model/v2/HBaseColumnFamilyDesc.java|  58 ++
 .../kylin/cube/model/v2/HBaseMappingDesc.java   |  96 +++
 .../kylin/cube/model/v2/HierarchyDesc.java  |  68 ++
 .../kylin/cube/model/v2/RowKeyColDesc.java  |  92 +++
 .../apache/kylin/cube/model/v2/RowKeyDesc.java  | 295 +++
 .../model/validation/CubeMetadataValidator.java |   9 +-
 .../cube/model/validation/ValidateContext.java  |   5 +-
 .../validation/rule/AggregationGroupRule.java   | 185 +
 .../rule/AggregationGroupSizeRule.java  |  66 --
 .../rule/IKylinValidationConstants.java |   6 +-
 .../validation/rule/MandatoryColumnRule.java|  75 --
 .../model/validation/rule/RowKeyAttrRule.java   |  27 +-
 .../cube/upgrade/V2/CubeDescUpgraderV2.java | 287 +++
 .../cube/upgrade/V2/CubeMetadataUpgradeV2.java  | 177 
 .../org/apache/kylin/cube/util/CubingUtils.java |  30 +-
 .../kylin/cube/AggregationGroupRuleTest.java| 127 +++
 .../cube/AggregationGroupSizeRuleTest.java  | 101 ---
 .../org/apache/kylin/cube/CubeManagerTest.java  |   2 +-
 .../kylin/cube/DictionaryManagerTest.java   |   4 +-
 .../kylin/cube/MandatoryColumnRuleTest.java |  57 --
 .../apache/kylin/cube/RowKeyAttrRuleTest.java   |  43 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  | 155 ++--
 .../apache/kylin/cube/cuboid/CuboidTest.java|  10 +-
 .../src/test/resources/data/TEST1_desc.json | 183 
 .../src/test/resources/data/TEST2_desc.json | 168 
 .../src/test/resources/data/TEST3_desc.json | 182 
 .../apache/kylin/dict/DictionaryManager.java|  10 +-
 .../kylin/job/manager/ExecutableManager.java|   4 +-
 .../kylin/metadata/model/MeasureDesc.java   |  16 +-
 .../kylin/engine/mr/BatchCubingJobBuilder.java  |   2 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java |   2 +-
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   3 +-
 .../mr/steps/MapContextGTRecordWriter.java  |   2 +-
 .../mr/steps/MergeCuboidFromStorageMapper.java  |   2 +-
 .../engine/mr/steps/MergeCuboidMapper.java  |   2 +-
 .../engine/mr/steps/MergeDictionaryStep.java|   2 +-
 .../kylin/engine/mr/steps/CubeSamplingTest.java |   3 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |   5 +-
 .../spark/cube/DefaultTupleConverter.java   |   3 +-
 .../apache/kylin/rest/service/CubeService.java  |   4 +-
 .../rest/controller/CubeControllerTest.java |   7 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |   2 +-
 .../hbase/cube/v1/CubeTupleConverter.java   |   6 +-
 .../storage/hbase/cube/v2/CubeStorageQuery.java |   2 +-
 .../hbase/cube/v2/CubeTupleConverter.java   |   3 +-
 .../kylin/storage/hbase/steps/BulkLoadJob.java  |   2 +-
 .../storage/hbase/steps/CubeHFileMapper.java|   2 +-
 .../storage/hbase/steps/CubeHTableUtil.java |   2 +-
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   2 +-
 .../storage/hbase/steps/HBaseMROutput2.java |   4 +-
 .../hbase/steps/HBaseMROutput2Transition.java   |   2 +-

[2/7] kylin git commit: KYLIN-242 refine cube desc to fit new aggregation group design

2015-12-04 Thread mahongbin 
KYLIN-242 refine cube desc to fit new aggregation group design


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af6b11a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af6b11a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af6b11a9

Branch: refs/heads/2.x-staging
Commit: af6b11a97c74eb45aba62fa65494bab98d33541c
Parents: 91b1f92
Author: honma 
Authored: Mon Nov 30 11:17:14 2015 +0800
Committer: honma 
Committed: Fri Dec 4 17:07:50 2015 +0800

--
 .../localmeta/cube_desc/test_kylin_cube_without_slr_desc.json | 3 +--
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 3 +--
 .../localmeta/cube_desc/test_streaming_table_cube_desc.json   | 3 ++-
 3 files changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af6b11a9/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 1e007c3..31c54a8 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -245,8 +245,7 @@
 "leaf_categ_id",
 "meta_categ_name",
 "categ_lvl3_name",
-"categ_lvl2_name",
-"lstg_format_name"
+"categ_lvl2_name"
   ]
 ]
   },

http://git-wip-us.apache.org/repos/asf/kylin/blob/af6b11a9/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 73a58f0..80ea23a 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -245,8 +245,7 @@
 "leaf_categ_id",
 "meta_categ_name",
 "categ_lvl3_name",
-"categ_lvl2_name",
-"lstg_format_name"
+"categ_lvl2_name"
   ]
 ]
   },

http://git-wip-us.apache.org/repos/asf/kylin/blob/af6b11a9/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index c9a6536..6f9b3a7 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -83,7 +83,7 @@
 "column": "DAY_START",
 "length": 0,
 "dictionary": "true",
-"mandatory": true
+"mandatory": false
   },
   {
 "column": "HOUR_START",
@@ -112,6 +112,7 @@
 ],
 "aggregation_groups": [
   [
+"DAY_START",
 "HOUR_START",
 "MINUTE_START",
 "SITE",

[3/7] kylin git commit: KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming joint semantic

2015-12-04 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 981e19a..2a1304a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -223,11 +223,11 @@ public class DictionaryManager {
 }
 }
 
-public DictionaryInfo buildDictionary(DataModelDesc model, String dict, 
TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws 
IOException {
+public DictionaryInfo buildDictionary(DataModelDesc model, boolean 
usingDict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) 
throws IOException {
 
 logger.info("building dictionary for " + col);
 
-TblColRef srcCol = decideSourceData(model, dict, col);
+TblColRef srcCol = decideSourceData(model, usingDict, col);
 String srcTable = srcCol.getTable();
 String srcColName = srcCol.getName();
 int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
@@ -259,9 +259,9 @@ public class DictionaryManager {
 /**
  * Decide a dictionary's source data, leverage PK-FK relationship.
  */
-public TblColRef decideSourceData(DataModelDesc model, String dict, 
TblColRef col) throws IOException {
+public TblColRef decideSourceData(DataModelDesc model, boolean usingDict, 
TblColRef col) throws IOException {
 // Note FK on fact table is supported by scan the related PK on lookup 
table
-if ("true".equals(dict) || "string".equals(dict) || 
"number".equals(dict) || "any".equals(dict)) {
+if (usingDict) {
 // FK on fact table and join type is inner, use PK from lookup 
instead
 if (model.isFactTable(col.getTable())) {
 TblColRef pkCol = model.findPKByFK(col, "inner");
@@ -270,7 +270,7 @@ public class DictionaryManager {
 }
 return col;
 } else
-throw new IllegalArgumentException("Unknown dictionary value: " + 
dict);
+throw new IllegalArgumentException("Not using Dictionary ");
 }
 
 private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
index ff92a50..d73fd26 100644
--- a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
@@ -299,6 +299,7 @@ public class ExecutableManager {
 
 private static AbstractExecutable parseTo(ExecutablePO executablePO) {
 if (executablePO == null) {
+logger.warn("executablePO is null");
 return null;
 }
 String type = executablePO.getType();
@@ -318,8 +319,7 @@ public class ExecutableManager {
 }
 return result;
 } catch (ReflectiveOperationException e) {
-logger.error("cannot parse this job:" + executablePO.getId(), e);
-return null;
+throw new IllegalStateException("cannot parse this job:" + 
executablePO.getId(), e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 618d25a..5d65771 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -31,8 +31,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class MeasureDesc {
 
-@JsonProperty("id")
-private int id;
+   
 @JsonProperty("name")
 private String name;
 @JsonProperty("function")
@@ -49,13 +48,7 @@ public class MeasureDesc {
 }
 }
 
-public int getId() {
-return id;
-}
-
-public void setId(int id) {
-this.id = id;
-}
+   
 
 public String getName() {
 return name;
@@ -103,11 +96,6 @@ public class MeasureDesc

[5/7] kylin git commit: KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming joint semantic

2015-12-04 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 278b59f..7c26e8f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -18,11 +18,15 @@
 
 package org.apache.kylin.cube.model;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.metadata.model.TblColRef;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
 
 /**
  * @author yangli9
@@ -31,37 +35,74 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class RowKeyColDesc {
 
+public enum ColEncodingType {
+DICT, FIXED_LEN
+}
+
+public class ColEncoding {
+public ColEncodingType type;
+public Object param;
+
+public ColEncoding(ColEncodingType type, Object param) {
+this.type = type;
+this.param = param;
+}
+}
+
 @JsonProperty("column")
 private String column;
-@JsonProperty("length")
-private int length;
-@JsonProperty("dictionary")
-private String dictionary;
-@JsonProperty("mandatory")
-private boolean mandatory = false;
+@JsonProperty("encoding")
+private String encoding;
 
 // computed
+private ColEncoding colEncoding;
 private int bitIndex;
 private TblColRef colRef;
 
-public String getDictionary() {
-return dictionary;
+public void init() {
+
+//dict or fix length?
+Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
+if (this.encoding.equalsIgnoreCase("dict")) {
+this.colEncoding = new ColEncoding(ColEncodingType.DICT, null);
+} else if (this.encoding.startsWith("fixed_length")) {
+int length = RowConstants.ROWKEY_COL_DEFAULT_LENGTH;
+if (this.encoding.indexOf(":") > 0) {
+length = 
Integer.parseInt(this.encoding.substring(this.encoding.indexOf(":") + 1));
+}
+this.colEncoding = new ColEncoding(ColEncodingType.FIXED_LEN, 
length);
+} else {
+throw new IllegalArgumentException("Not supported row key col 
encoding:" + this.encoding);
+}
+}
+
+public String getEncoding() {
+return encoding;
+}
+
+public void setEncoding(String encoding) {
+this.encoding = encoding;
 }
 
 public String getColumn() {
 return column;
 }
 
-void setColumn(String column) {
+public void setColumn(String column) {
 this.column = column;
 }
 
-public int getLength() {
-return length;
+public boolean isUsingDictionary() {
+return this.colEncoding.type == ColEncodingType.DICT;
+
 }
 
-public boolean isMandatory() {
-return mandatory;
+public int getLength() {
+if (this.colEncoding.type == ColEncodingType.FIXED_LEN) {
+return (Integer) this.colEncoding.param;
+} else {
+return 0;
+}
 }
 
 public int getBitIndex() {
@@ -80,13 +121,9 @@ public class RowKeyColDesc {
 this.colRef = colRef;
 }
 
-public void setDictionary(String dictionary) {
-this.dictionary = dictionary;
-}
-
 @Override
 public String toString() {
-return "RowKeyColDesc [column=" + column + ", length=" + length + ", 
dictionary=" + dictionary + ", mandatory=" + mandatory + "]";
+return Objects.toStringHelper(this).add("column", 
column).add("encoding", encoding).toString();
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index abec436..7254df0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -18,139 +18,65 @@
 
 package org.apache.kylin.cube.model;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import

[4/7] kylin git commit: KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming joint semantic

2015-12-04 Thread mahongbin 
http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
new file mode 100644
index 000..17f5b2d
--- /dev/null
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model.validation.rule;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ResultLevel;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+
+/**
+ *  find forbid overlaps in each AggregationGroup
+ *  the include dims in AggregationGroup must contain all mandatory, hierarchy 
and joint
+ */
+public class AggregationGroupRule implements IValidatorRule {
+
+@Override
+public void validate(CubeDesc cube, ValidateContext context) {
+inner(cube, context);
+}
+
+private int count(String[][] input) {
+if (input == null) {
+return 0;
+} else {
+int count = 0;
+for (String[] x : input) {
+count += count(x);
+}
+return count;
+}
+}
+
+private int count(String[] input) {
+return input == null ? 0 : input.length;
+}
+
+private void inner(CubeDesc cube, ValidateContext context) {
+int maxSize = getMaxAgrGroupSize();
+
+int index = 0;
+for (AggregationGroup agg : cube.getAggregationGroups()) {
+if (agg.getIncludes() == null) {
+context.addResult(ResultLevel.ERROR, "Aggregation group " + 
index + " includes field not set");
+continue;
+}
+
+if (agg.getSelectRule() == null) {
+context.addResult(ResultLevel.ERROR, "Aggregation group " + 
index + " select rule field not set");
+continue;
+}
+
+Set includeDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+if (agg.getIncludes() != null) {
+for (String include : agg.getIncludes()) {
+includeDims.add(include);
+}
+}
+
+Set mandatoryDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+if (agg.getSelectRule().mandatory_dims != null) {
+for (String m : agg.getSelectRule().mandatory_dims) {
+mandatoryDims.add(m);
+}
+}
+
+Set hierarchyDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+if (agg.getSelectRule().hierarchy_dims != null) {
+for (String[] ss : agg.getSelectRule().hierarchy_dims) {
+for (String s : ss)
+hierarchyDims.add(s);
+}
+}
+
+Set jointDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+if (agg.getSelectRule().joint_dims != null) {
+for (String[] ss : agg.getSelectRule().joint_dims) {
+for (String s : ss)
+jointDims.add(s);
+}
+}
+
+if (!includeDims.containsAll(mandatoryDims) || 
!includeDims.containsAll(hierarchyDims) || !includeDims.containsAll(jointDims)) 
{
+context.addResult(ResultLevel.ERROR, "Aggregation group " + 
index + " Include dims not containing all the used dims");
+continue;
+}
+
+Set normalDims = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+normalDims.addAll(includeDims);
+

[1/7] kylin git commit: minor: give find command -L option to follow all symbolic links

2015-12-04 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 4afd144af -> 5d16f9bf6


minor: give find command -L option to follow all symbolic links


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/91b1f922
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/91b1f922
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/91b1f922

Branch: refs/heads/2.x-staging
Commit: 91b1f922314fbc113709de41add140f4e01382b0
Parents: 4afd144
Author: honma 
Authored: Mon Nov 23 13:36:49 2015 +0800
Committer: honma 
Committed: Fri Dec 4 17:02:02 2015 +0800

--
 build/bin/cleanup_streaming_files.sh | 2 +-
 build/bin/find-hive-dependency.sh| 4 ++--
 build/bin/sample.sh  | 2 +-
 build/bin/streaming_rolllog.sh   | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/91b1f922/build/bin/cleanup_streaming_files.sh
--
diff --git a/build/bin/cleanup_streaming_files.sh 
b/build/bin/cleanup_streaming_files.sh
index 3837a2d..4fc9f0e 100644
--- a/build/bin/cleanup_streaming_files.sh
+++ b/build/bin/cleanup_streaming_files.sh
@@ -8,7 +8,7 @@ fi
 
 cd $KYLIN_HOME/logs
 
-for pidfile in `find . -name "$1_1*"`
+for pidfile in `find -L . -name "$1_1*"`
 do
 pidfile=`echo "$pidfile" | cut -c 3-`
 echo "pidfile:$pidfile"

http://git-wip-us.apache.org/repos/asf/kylin/blob/91b1f922/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh 
b/build/bin/find-hive-dependency.sh
index 47f0fbc..5994dda 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -55,7 +55,7 @@ else
 hcatalog_home=${HCAT_HOME}
 fi
 
-hcatalog=`find ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | 
grep -m 1 -v 'Permission denied'`
+hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 
| grep -m 1 -v 'Permission denied'`
 
 if [ -z "$hcatalog" ]
 then
@@ -64,7 +64,7 @@ then
 fi
 
 
-hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' 
-printf '%p:' | sed 's/:$//'`
+hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name 
'*calcite*' -printf '%p:' | sed 's/:$//'`
 hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
 echo "hive dependency: $hive_dependency"
 export hive_dependency

http://git-wip-us.apache.org/repos/asf/kylin/blob/91b1f922/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 6931457..a71d43f 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 dir=$(dirname ${0})
 source ${dir}/check-env.sh
-job_jar=`find ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
+job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
 echo "Going to create sample tables in hive..."
 cd ${KYLIN_HOME}/sample_cube/data
 hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }

http://git-wip-us.apache.org/repos/asf/kylin/blob/91b1f922/build/bin/streaming_rolllog.sh
--
diff --git a/build/bin/streaming_rolllog.sh b/build/bin/streaming_rolllog.sh
index 9380f49..1fa5bb7 100644
--- a/build/bin/streaming_rolllog.sh
+++ b/build/bin/streaming_rolllog.sh
@@ -7,6 +7,6 @@ KYLIN_LOG_HOME=${KYLIN_HOME}/logs
 cd ${KYLIN_LOG_HOME}
 timestamp=`date +%Y_%m_%d_%H_%M_%S`
 tarfile=logs_archived_at_${timestamp}.tar
-files=`find . ! -name '*.tar' -type f -mtime +1` # keep two days' log
+files=`find -L . ! -name '*.tar' -type f -mtime +1` # keep two days' log
 echo ${files} | xargs tar -cvf ${tarfile}
 echo ${files} | xargs rm
\ No newline at end of file

kylin git commit: update website, add TM mark, added 2 companies on powered by page.

2015-12-04 Thread lukehan 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 95bc7dfa6 -> 9457552e0


update website, add TM mark, added 2 companies on powered by page.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9457552e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9457552e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9457552e

Branch: refs/heads/1.x-staging
Commit: 9457552e07555a67a5b8c0389981cb5eadd59214
Parents: 95bc7df
Author: lukehan 
Authored: Fri Dec 4 22:50:44 2015 +0800
Committer: lukehan 
Committed: Fri Dec 4 22:50:44 2015 +0800

--
 website/_data/contributors.yml | 30 --
 website/community/poweredby.md |  5 +
 website/download/index.cn.md   |  2 +-
 website/download/index.md  |  2 +-
 website/index.cn.md|  7 +++
 website/index.md   |  7 +++
 6 files changed, 37 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/_data/contributors.yml
--
diff --git a/website/_data/contributors.yml b/website/_data/contributors.yml
index dfc3d9b..17b9e38 100644
--- a/website/_data/contributors.yml
+++ b/website/_data/contributors.yml
@@ -39,7 +39,7 @@
   githubId: julianhyde
   org: Hortonworks
   role: Mentor
-- name: Luke Han (韩卿) 
+- name: Luke Han (韩卿) (Chair)
   apacheId: lukehan
   githubId: lukehan
   org: eBay
@@ -74,12 +74,12 @@
   apacheId: xduo
   githubId: xduo
   org: Alipay
-  role: PMC
+  role: Emeritus PMC
 - name: Ankur Bansal
   apacheId: abansal
   githubId: abansal
   org: eBay
-  role: PMC
+  role: Emeritus PMC
 - name: Xu Jiang (蒋旭)
   apacheId: jiangxu
   githubId: jiangxuchina
@@ -94,12 +94,30 @@
   apacheId: 
   githubId: gaodayue
   org: Meituan
-  role: committer
+  role: PMC
 - name: Hua Huang (黄桦)
   apacheId: 
   githubId: superhua
   org: MiningLAMP
+  role: PMC 
+- name: Dong Li (李栋)
+  apacheId: 
+  githubId: 
+  org: eBay
+  role: committer   
+- name: Xiaoyu Wang (王晓雨)
+  apacheId: 
+  githubId: 
+  org: JD.com
   role: committer 
-
-
+- name: Dong Li (李栋)
+  apacheId: 
+  githubId: lidongsjtu
+  org: eBay
+  role: committer 
+- name: Yerui Sun (孙叶锐)
+  apacheId: sunyerui
+  githubId: 
+  org: Meituan
+  role: PMC
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/community/poweredby.md
--
diff --git a/website/community/poweredby.md b/website/community/poweredby.md
index 10d1b93..27448cc 100644
--- a/website/community/poweredby.md
+++ b/website/community/poweredby.md
@@ -25,4 +25,9 @@ __Companies & Organizations__
 * DreamSoft is using Apache Kylin as Big Data Analytics Engine for their 
customers coming from Retail, Manufacturing, Finance and others.
 * [Exponential](http://www.exponential.com)
* Using Kylin as a component in it's data platform LAMBDA to power 
inventory, campaign, behavior and demand analysis for advertising. It has been 
great at handling the size and speed constraints of old gen systems. With the 
first version already in production, we are looking forward to pushing more 
data into kylin and achieve near real time updates. 
+* [Baidu Map](http://map.baidu.com/)  (_NASDAQ: BIDU_)
+   * Apache Kylin is used for analysis Baidu Map's huge amounts of data 
about users, channels and etc, it performs quite fast with its own 
pre-calculation and index technologies along with the cubes.
+* [NetEase](http://www.163.com/)  (_NASDAQ: NTES_)
+   * Apache has been introduced as an OLAP engine for the data platform at 
NetEase, powering data analysis for various products & business with great 
performance.
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 81afb2e..ff4cacb 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -4,7 +4,7 @@ title: 下载
 ---
 
 __最新发布(源代码)__  
-最新发布的Apache Kylin可以从ASF网站下载：:
+最新发布的Apache Kylin™可以从ASF网站下载：:
 
 * [Apache Kylin 
v1.1.1-incubating](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.1.1-incubating/)
 * [发布日志](http://kylin.apache.org/docs/release_notes.html)

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/download/index.md
--
diff --git a/website/download/index.md b/website/download/index.md
index e834ccf..eed1a4b 100644
--- a/website/download/index.md
+++

[16/21] kylin git commit: KYLIN-976 Extract fillTuple() methods

2015-12-04 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
--
diff --git 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
index c395fd9..bde8dd2 100644
--- 
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
+++ 
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
@@ -26,7 +26,7 @@ import java.util.List;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilterSerializer;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.Tuple;
 import org.junit.Test;
 
 /**

[10/21] kylin git commit: KYLIN-976 Add ingester; Build part done, in-mem cube test pass

2015-12-04 Thread liyang 
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
index ed0a166..832584c 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
@@ -33,10 +33,10 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.junit.AfterClass;
@@ -58,7 +58,7 @@ public class DoggedCubeBuilderTest extends 
LocalFileMetadataTestCase {
 
 private static CubeInstance cube;
 private static String flatTable;
-private static Map dictionaryMap;
+private static Map dictionaryMap;
 
 @BeforeClass
 public static void before() throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
 
b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
index f853b08..ab87c2b 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
@@ -33,13 +33,13 @@ import java.util.concurrent.Future;
 import org.apache.commons.io.FileUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.dict.DictionaryGenerator;
 import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
 import org.apache.kylin.gridtable.GTRecord;
@@ -66,7 +66,7 @@ public class InMemCubeBuilderTest extends 
LocalFileMetadataTestCase {
 
 private static CubeInstance cube;
 private static String flatTable;
-private static Map dictionaryMap;
+private static Map dictionaryMap;
 
 @BeforeClass
 public static void before() throws IOException {
@@ -166,8 +166,8 @@ public class InMemCubeBuilderTest extends 
LocalFileMetadataTestCase {
 queue.put(new ArrayList(0));
 }
 
-static Map getDictionaryMap(CubeInstance cube, 
String flatTable) throws IOException {
-Map result = Maps.newHashMap();
+static Map getDictionaryMap(CubeInstance 
cube, String flatTable) throws IOException {
+Map result = Maps.newHashMap();
 CubeDesc desc = cube.getDescriptor();
 CubeJoinedFlatTableDesc flatTableDesc = new 
CubeJoinedFlatTableDesc(desc, null);
 int nColumns = flatTableDesc.getColumnList().size();
@@ -178,7 +178,7 @@ public class InMemCubeBuilderTest extends 
LocalFileMetadataTestCase {
 if (desc.getRowkey().isUseDictionary(col)) {
 logger.info("Building dictionary for " + col);
 List valueList = readValueList(flatTable, nColumns, 
flatTableDesc.getRowKeyColumnIndexes()[c]);
-Dictionary dict = 
DictionaryGenerator.buildDictionaryFromValueEnumerator(col.getType(), new 
IterableDictionaryValueEnumerator(valueList));
+Dictionary dict = 
DictionaryGenerator.buildDictionaryFromValueEnumerator(col.getType(), new 
IterableDictionaryValueEnumerator(valueList));
 result.put(col, dict);
 }
 }
@@ -192,7 +192,7 @@ public class InMemCubeBuilderTest extends 
LocalFileMetadataTestCase {
 TblColRef literalCol = func.getTopNLiteralColumn();
 logger.info("Building dictionary for " + literalCol);
 List valueList = readValueList(flatTable, nColumns, 
literalColIdx);
-Dictionary dict =

[07/21] kylin git commit: KYLIN-976 AggregationType interface, serializer and aggregator

2015-12-04 Thread liyang 
KYLIN-976 AggregationType interface, serializer and aggregator


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2d82d38
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2d82d38
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2d82d38

Branch: refs/heads/2.x-staging
Commit: b2d82d3863a9aaa68a3b370d6f0cb598e07b4ce5
Parents: 836f233
Author: Li, Yang 
Authored: Wed Nov 25 11:13:18 2015 +0800
Committer: Yang Li 
Committed: Sat Dec 5 06:48:43 2015 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   |   2 +-
 .../streaming/StreamingTableDataGenerator.java  |   2 +-
 .../common/datatype/BigDecimalSerializer.java   | 109 +++
 .../apache/kylin/common/datatype/DataType.java  | 289 ++
 .../common/datatype/DataTypeSerializer.java | 108 +++
 .../common/datatype/DateTimeSerializer.java |  62 
 .../kylin/common/datatype/DoubleMutable.java|  68 +
 .../kylin/common/datatype/DoubleSerializer.java |  81 +
 .../kylin/common/datatype/LongMutable.java  |  70 +
 .../kylin/common/datatype/LongSerializer.java   |  88 ++
 .../kylin/common/datatype/StringSerializer.java |  54 
 .../kylin/common/debug/BackdoorToggles.java |   5 -
 .../kylin/common/topn/DoublyLinkedList.java |   2 -
 .../apache/kylin/common/util/DoubleMutable.java |  68 -
 .../apache/kylin/common/util/LongMutable.java   |  70 -
 .../datatype/BigDecimalSerializerTest.java  |  51 
 .../apache/kylin/aggregation/Aggregation.java   |  42 ---
 .../kylin/aggregation/AggregationType.java  |  97 ++
 .../kylin/aggregation/DataTypeSerializer.java   | 118 
 .../kylin/aggregation/IAggregationFactory.java  |   6 +
 .../kylin/aggregation/MeasureAggregator.java|  61 +---
 .../apache/kylin/aggregation/MeasureCodec.java  |   1 +
 .../aggregation/basic/BasicAggregation.java | 128 
 .../basic/BasicAggregationFactory.java  |  31 ++
 .../aggregation/basic/BigDecimalSerializer.java | 111 ---
 .../aggregation/basic/DateTimeSerializer.java   |  65 
 .../aggregation/basic/DoubleMaxAggregator.java  |   2 +-
 .../aggregation/basic/DoubleMinAggregator.java  |   2 +-
 .../aggregation/basic/DoubleSerializer.java |  84 --
 .../aggregation/basic/DoubleSumAggregator.java  |   2 +-
 .../aggregation/basic/LongMaxAggregator.java|   2 +-
 .../aggregation/basic/LongMinAggregator.java|   2 +-
 .../kylin/aggregation/basic/LongSerializer.java |  91 --
 .../aggregation/basic/LongSumAggregator.java|   2 +-
 .../aggregation/basic/StringSerializer.java |  56 
 .../kylin/aggregation/hllc/HLLCAggregation.java |  78 +
 .../hllc/HLLCAggregationFactory.java|  35 +++
 .../kylin/aggregation/hllc/HLLCSerializer.java  |   4 +-
 .../kylin/aggregation/hllc/LDCAggregator.java   |   2 +-
 .../kylin/aggregation/topn/TopNAggregation.java |  76 +
 .../topn/TopNAggregationFactory.java|  35 +++
 .../aggregation/topn/TopNCounterSerializer.java |  12 +-
 .../kylin/cube/gridtable/CubeCodeSystem.java|   4 +-
 .../kylin/cube/gridtable/CubeGridTable.java |   1 -
 .../gridtable/CuboidToGridTableMapping.java |   2 +-
 .../cube/gridtable/TrimmedCubeCodeSystem.java   |   3 +-
 .../cube/inmemcubing/InMemCubeBuilder.java  |   2 +-
 .../InMemCubeBuilderInputConverter.java |   2 +-
 .../apache/kylin/cube/kv/RowKeyColumnOrder.java |   2 +-
 .../model/validation/rule/FunctionRule.java |   2 +-
 .../java/org/apache/kylin/gridtable/GTInfo.java |   2 +-
 .../kylin/gridtable/GTSampleCodeSystem.java |   2 +-
 .../apache/kylin/gridtable/UnitTestSupport.java |  14 +-
 .../basic/BigDecimalSerializerTest.java |  53 
 .../topn/TopNCounterSerializerTest.java |   9 +-
 .../gridtable/AggregationCacheMemSizeTest.java  |   4 +-
 .../kylin/gridtable/DictGridTableTest.java  |  16 +-
 .../kylin/gridtable/SimpleGridTableTest.java|   2 +-
 .../gridtable/SimpleInvertedIndexTest.java  |  13 +-
 .../metadata/measure/MeasureCodecTest.java  |   4 +-
 .../apache/kylin/dict/DictionaryGenerator.java  |   9 +-
 .../apache/kylin/dict/DictionaryManager.java|   4 +-
 .../apache/kylin/dict/NumberDictionaryTest.java |   9 +-
 .../filter/TimeConditionLiteralsReplacer.java   |   2 +-
 .../apache/kylin/metadata/model/ColumnDesc.java |   9 +-
 .../apache/kylin/metadata/model/DataType.java   | 295 ---
 .../kylin/metadata/model/FunctionDesc.java  |  11 +-
 .../apache/kylin/metadata/model/TblColRef.java  |   5 +-
 .../metadata/realization/SQLDigestUtil.java |   6 +-
 .../org/apache/kylin/storage/tuple/Tuple.java   |   4 +-
 .../kylin/engine/mr/steps/CubeReducerTest.java  |   2 +-
 .../engine/mr/steps/MergeCuboidMapperTest.java  |  15 +-
 .../invertedindex/index/RawTableRecord.java |   2 +-

[19/21] kylin git commit: KYLIN-976 Clean up, remove all explict check on TopN & HLLC

2015-12-04 Thread liyang 
KYLIN-976 Clean up, remove all explict check on TopN & HLLC


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f50c17a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f50c17a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f50c17a

Branch: refs/heads/2.x-staging
Commit: 7f50c17a5202be9508c21d9cc301d231c8b5843e
Parents: 636aeec
Author: Li, Yang 
Authored: Fri Dec 4 16:38:04 2015 +0800
Committer: Yang Li 
Committed: Sat Dec 5 06:55:39 2015 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  16 ++-
 .../apache/kylin/common/topn/TopNCounter.java   |   2 +-
 .../kylin/common/util/CliCommandExecutor.java   |   4 +-
 .../java/org/apache/kylin/common/util/Pair.java |   3 +-
 .../kylin/common/topn/TopNCounterTest.java  |   4 +-
 .../kylin/cube/CubeCapabilityChecker.java   |   5 -
 .../org/apache/kylin/cube/CubeDescManager.java  |  15 ++-
 .../org/apache/kylin/cube/CubeInstance.java |   3 -
 .../java/org/apache/kylin/cube/CubeManager.java |   4 +-
 .../kylin/cube/gridtable/CubeCodeSystem.java|   2 +-
 .../gridtable/CuboidToGridTableMapping.java |  30 +
 .../cube/gridtable/TrimmedCubeCodeSystem.java   |   2 +-
 .../InMemCubeBuilderInputConverter.java |   3 +-
 .../cube/inmemcubing/InMemCubeBuilderUtils.java |  30 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  11 +-
 .../model/validation/rule/FunctionRule.java |   4 -
 .../kylin/gridtable/GTSampleCodeSystem.java |   2 +-
 .../cube/inmemcubing/InMemCubeBuilderTest.java  |  25 ++--
 .../apache/kylin/dict/lookup/LookupTable.java   |   2 +-
 .../apache/kylin/measure/IMeasureFactory.java   |  24 
 .../apache/kylin/measure/MeasureAggregator.java |   6 +-
 .../kylin/measure/MeasureAggregators.java   |   2 +-
 .../org/apache/kylin/measure/MeasureType.java   |  50 ++--
 .../kylin/measure/MeasureTypeFactory.java   |  89 ++
 .../measure/basic/BasicMeasureFactory.java  |  32 -
 .../kylin/measure/basic/BasicMeasureType.java   |  74 +++-
 .../measure/hllc/HLLCAggregationFactory.java|  36 --
 .../kylin/measure/hllc/HLLCMeasureType.java |  76 
 .../kylin/measure/hllc/HLLCSerializer.java  |   3 +-
 .../kylin/measure/hllc/LDCAggregator.java   |  63 --
 .../kylin/measure/topn/TopNMeasureFactory.java  |  36 --
 .../kylin/measure/topn/TopNMeasureType.java | 116 +--
 .../kylin/metadata/datatype/DataType.java   |  33 --
 .../metadata/datatype/DataTypeSerializer.java   |  38 +++---
 .../kylin/metadata/model/FunctionDesc.java  |  65 ++-
 .../kylin/metadata/tuple/TeeTupleIterator.java  |   5 -
 .../org/apache/kylin/metadata/tuple/Tuple.java  |  12 +-
 .../kylin/storage/translate/HBaseKeyRange.java  |   2 +-
 .../apache/kylin/engine/mr/DFSFileTable.java|   2 +-
 .../engine/mr/steps/BaseCuboidMapperBase.java   |   3 +-
 .../apache/kylin/engine/mr/steps/CuboidJob.java |   5 +-
 .../mr/steps/MergeCuboidFromStorageMapper.java  |   2 +-
 .../engine/mr/steps/MergeCuboidMapper.java  |   2 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |  11 +-
 .../measure/FixedLenMeasureCodec.java   |   3 +-
 .../cube/v1/CubeSegmentTopNTupleIterator.java   |  86 --
 .../hbase/cube/v1/CubeSegmentTupleIterator.java |   2 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  46 ++--
 .../hbase/cube/v1/CubeTupleConverter.java   |  60 +-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  14 +--
 .../observer/ObserverAggregators.java   |  20 +++-
 .../hbase/cube/v1/filter/FuzzyRowFilterV2.java  |   6 +-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |   4 +-
 .../storage/hbase/cube/v2/CubeStorageQuery.java |  67 +++
 .../hbase/cube/v2/CubeTupleConverter.java   |  65 +--
 .../hbase/cube/v2/HBaseReadonlyStore.java   |   1 -
 .../v2/SequentialCubeTopNTupleIterator.java |  68 ---
 .../cube/v2/SequentialCubeTupleIterator.java|   2 +-
 .../endpoint/EndpointAggregators.java   |   7 +-
 .../endpoint/EndpointTupleIterator.java |   3 +-
 .../storage/hbase/steps/CreateHTableJob.java|   7 +-
 .../storage/hbase/steps/HBaseMROutput2.java |   2 +-
 .../hbase/steps/HBaseMROutput2Transition.java   |   2 +-
 .../storage/hbase/steps/RowValueDecoder.java|  21 +---
 .../hbase/util/GridTableHBaseBenchmark.java |   2 +-
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |   8 +-
 66 files changed, 473 insertions(+), 977 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git