date:20161219

[2/2] kylin git commit: KYLIN-2287 minor, UI enhancement

2016-12-19 Thread zhongjian

KYLIN-2287 minor, UI enhancement


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c72e2320
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c72e2320
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c72e2320

Branch: refs/heads/master
Commit: c72e2320238edeb2e7279dc4dc152493837e8843
Parents: 64a0a59
Author: zhongjian 
Authored: Tue Dec 20 15:43:12 2016 +0800
Committer: zhongjian 
Committed: Tue Dec 20 15:43:12 2016 +0800

--
 webapp/app/less/app.less | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c72e2320/webapp/app/less/app.less
--
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index 5ef5297..85ad937 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -72,6 +72,14 @@ body {
   min-width: 110px
 }
 
+.cube_model_trees .dropdown-menu{
+  min-width: 60px;
+}
+
+.cube_model_trees a{
+  margin-right:0px !important;
+}
+
 .login {
   position: fixed;
   width: 325px;

[1/2] kylin git commit: KYLIN-2287 Speed up model and cube list load in Web

2016-12-19 Thread zhongjian

Repository: kylin
Updated Branches:
  refs/heads/master f50c0c873 -> c72e23202


KYLIN-2287 Speed up model and cube list load in Web

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64a0a594
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64a0a594
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64a0a594

Branch: refs/heads/master
Commit: 64a0a5943c034ea6f9cdc6d179d8be9da64fe5ac
Parents: f50c0c8
Author: kangkaisen 
Authored: Wed Dec 14 20:08:14 2016 +0800
Committer: zhongjian 
Committed: Tue Dec 20 15:12:54 2016 +0800

--
 webapp/app/js/controllers/cubeSchema.js |  25 +-
 webapp/app/js/controllers/cubes.js  | 318 ++-
 webapp/app/js/controllers/models.js |  54 ++--
 webapp/app/js/model/cubeListModel.js|   4 -
 webapp/app/js/model/modelsManager.js|  49 +---
 webapp/app/partials/cubes/cubes.html|  12 +-
 webapp/app/partials/models/models_tree.html |  16 +-
 7 files changed, 215 insertions(+), 263 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/controllers/cubeSchema.js
--
diff --git a/webapp/app/js/controllers/cubeSchema.js 
b/webapp/app/js/controllers/cubeSchema.js
index 357b6af..af8ee7c 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -45,18 +45,6 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, 
QueryService, UserServic
 $scope.state = {mode: "view"};
 }
 
-  var queryParam = {offset: 0, limit: 65535};
-
-  CubeService.list(queryParam, function (all_cubes) {
-if($scope.allCubes.length > 0){
-  $scope.allCubes.splice(0,$scope.allCubes.length);
-}
-
-for (var i = 0; i < all_cubes.length; i++) {
-  $scope.allCubes.push(all_cubes[i].name.toUpperCase());
-}
-  });
-
 $scope.$watch('cubeMetaFrame', function (newValue, oldValue) {
 if(!newValue){
 return;
@@ -198,6 +186,19 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, 
QueryService, UserServic
 };
 
   $scope.check_cube_info = function(){
+
+var queryParam = {offset: 0, limit: 65535};
+
+CubeService.list(queryParam, function (all_cubes) {
+  if($scope.allCubes.length > 0){
+$scope.allCubes.splice(0,$scope.allCubes.length);
+  }
+
+  for (var i = 0; i < all_cubes.length; i++) {
+$scope.allCubes.push(all_cubes[i].name.toUpperCase());
+  }
+});
+
 if(($scope.state.mode === "edit") 
&&$scope.cubeMode=="addNewCube"&&($scope.allCubes.indexOf($scope.cubeMetaFrame.name.toUpperCase())
 >= 0)){
   SweetAlert.swal('Oops...', "The cube named [" + 
$scope.cubeMetaFrame.name.toUpperCase() + "] already exists", 'warning');
   return false;

http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index b672d3f..cbdbb7d 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -79,43 +79,6 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, 
$routeParams, $location,
 
   return CubeList.list(queryParam).then(function (resp) {
 angular.forEach($scope.cubeList.cubes,function(cube,index){
-  cube.streaming = false;
-  CubeDescService.query({cube_name: cube.name}, {}, function (detail) {
-if (detail.length > 0 && detail[0].hasOwnProperty("name")) {
-  cube.detail = detail[0];
-  
ModelService.list({projectName:$scope.projectModel.selectedProject,modelName:cube.detail.model_name},
 function (_models) {
-if(_models && _models.length){
-  for(var i=0;i<=_models.length;i++){
-if(_models[i].name == cube.detail.model_name){
-  cube.model = _models[i];
-  var factTable = cube.model.fact_table;
-  TableService.get({tableName:factTable},function(table){
-if(table && table.source_type == 1){
-  cube.streaming = true;
-}
-  })
-  break;
-}
-  }
-}
-
-  })
-  //cube.model = modelsManager.getModel(cube.detail.model_name);
-
-  defer.resolve(cube.detail);
-
-} else {
-  SweetAlert.swal('Oops...', "No cube detail info loaded.", 
'error');
-}
-  }, function (e) {
-if (e.data &&

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-19 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/74b5f919/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/74b5f919/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-12-19 Thread billyliu

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bed831cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bed831cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bed831cc

Branch: refs/heads/yang22-cdh5.7
Commit: bed831cc34e7fea05ea5d300dbb2787e07289077
Parents: 74b5f91
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Tue Dec 20 11:28:04 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bed831cc/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/bed831cc/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bed831cc/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-19 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74b5f919
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74b5f919
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74b5f919

Branch: refs/heads/yang22-cdh5.7
Commit: 74b5f9198a936906e470ccb38528fb51ad6312c1
Parents: 7eb3d7c
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Tue Dec 20 11:25:06 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/74b5f919/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/4] kylin git commit: minor, fix sample model [Forced Update!]

2016-12-19 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 b0c90d4af -> bed831cc3 (forced update)


minor, fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7eb3d7c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7eb3d7c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7eb3d7c9

Branch: refs/heads/yang22-cdh5.7
Commit: 7eb3d7c9dbca29627cc00ac7ff921a7c9521ec4e
Parents: bcf2d0e
Author: Li Yang 
Authored: Tue Dec 20 11:03:08 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 20 11:03:08 2016 +0800

--
 .../sample_cube/template/model_desc/kylin_sales_model.json | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7eb3d7c9/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 3c4fa5a..2da2246 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -24,13 +24,11 @@
 "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
-"columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]
+"columns" : [ "TRANS_ID", "PART_DT", "LSTG_FORMAT_NAME", "LEAF_CATEG_ID", 
"LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "USER_ID", "REGION" ]
   } ],
   "metrics": [
 "PRICE",
-"ITEM_COUNT",
-"SELLER_ID",
-"LSTG_FORMAT_NAME"
+"ITEM_COUNT"
   ],
   "last_modified" : 1422435345362,
   "fact_table" : "DEFAULT.KYLIN_SALES",

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-19 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74b5f919
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74b5f919
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74b5f919

Branch: refs/heads/yang22-hbase1.x
Commit: 74b5f9198a936906e470ccb38528fb51ad6312c1
Parents: 7eb3d7c
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Tue Dec 20 11:25:06 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/74b5f919/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/3] kylin git commit: minor, fix sample model [Forced Update!]

2016-12-19 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 7f82104e9 -> 74b5f9198 (forced update)


minor, fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7eb3d7c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7eb3d7c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7eb3d7c9

Branch: refs/heads/yang22-hbase1.x
Commit: 7eb3d7c9dbca29627cc00ac7ff921a7c9521ec4e
Parents: bcf2d0e
Author: Li Yang 
Authored: Tue Dec 20 11:03:08 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 20 11:03:08 2016 +0800

--
 .../sample_cube/template/model_desc/kylin_sales_model.json | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7eb3d7c9/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 3c4fa5a..2da2246 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -24,13 +24,11 @@
 "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
-"columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]
+"columns" : [ "TRANS_ID", "PART_DT", "LSTG_FORMAT_NAME", "LEAF_CATEG_ID", 
"LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "USER_ID", "REGION" ]
   } ],
   "metrics": [
 "PRICE",
-"ITEM_COUNT",
-"SELLER_ID",
-"LSTG_FORMAT_NAME"
+"ITEM_COUNT"
   ],
   "last_modified" : 1422435345362,
   "fact_table" : "DEFAULT.KYLIN_SALES",

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-19 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/74b5f919/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/74b5f919/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

kylin git commit: minor, fix sample model

2016-12-19 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang22 bcf2d0ef2 -> 7eb3d7c9d


minor, fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7eb3d7c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7eb3d7c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7eb3d7c9

Branch: refs/heads/yang22
Commit: 7eb3d7c9dbca29627cc00ac7ff921a7c9521ec4e
Parents: bcf2d0e
Author: Li Yang 
Authored: Tue Dec 20 11:03:08 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 20 11:03:08 2016 +0800

--
 .../sample_cube/template/model_desc/kylin_sales_model.json | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7eb3d7c9/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 3c4fa5a..2da2246 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -24,13 +24,11 @@
 "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
-"columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]
+"columns" : [ "TRANS_ID", "PART_DT", "LSTG_FORMAT_NAME", "LEAF_CATEG_ID", 
"LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID", "USER_ID", "REGION" ]
   } ],
   "metrics": [
 "PRICE",
-"ITEM_COUNT",
-"SELLER_ID",
-"LSTG_FORMAT_NAME"
+"ITEM_COUNT"
   ],
   "last_modified" : 1422435345362,
   "fact_table" : "DEFAULT.KYLIN_SALES",

kylin git commit: try pass UT

2016-12-19 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2294 21531d734 -> 614b6ecb9


try pass UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/614b6ecb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/614b6ecb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/614b6ecb

Branch: refs/heads/KYLIN-2294
Commit: 614b6ecb98cde6d893f28287433b00efe663426b
Parents: 21531d7
Author: Yang Li 
Authored: Mon Dec 19 20:52:53 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 19 20:52:53 2016 +0800

--
 .../validation/rule/DictionaryRuleTest.java |   6 +-
 .../kylin/cube/project/ProjectManagerTest.java  |   4 +-
 .../localmeta/cube_desc/ci_inner_join_cube.json | 294 +++
 .../localmeta/cube_desc/ci_left_join_cube.json  | 294 +++
 4 files changed, 593 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/614b6ecb/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
 
b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index b6e0bcb..c272bb7 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -72,8 +72,8 @@ public class DictionaryRuleTest extends 
LocalFileMetadataTestCase {
 
 @Test
 public void testBadDesc() throws IOException {
-testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, 
DictionaryDesc.create("USER_ID", null, "FakeBuilderClass"));
-testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, 
DictionaryDesc.create("USER_ID", null, 
GlobalDictionaryBuilder.class.getName()));
+testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, 
DictionaryDesc.create("ORDER_ID", null, "FakeBuilderClass"));
+testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, 
DictionaryDesc.create("ORDER_ID", null, 
GlobalDictionaryBuilder.class.getName()));
 }
 
 @Test
@@ -89,7 +89,7 @@ public class DictionaryRuleTest extends 
LocalFileMetadataTestCase {
 @Test
 public void testBadDesc4() throws IOException {
 testDictionaryDesc(ERROR_TRANSITIVE_REUSE,
-DictionaryDesc.create("lstg_site_id", "USER_ID", null),
+DictionaryDesc.create("lstg_site_id", "SELLER_ID", null),
 DictionaryDesc.create("price", "lstg_site_id", null));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/614b6ecb/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
index 9728e52..a0fb4a3 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/project/ProjectManagerTest.java
@@ -147,7 +147,7 @@ public class ProjectManagerTest extends 
LocalFileMetadataTestCase {
 ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
 CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
 
-CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
+CubeInstance cube = cubeMgr.getCube("ci_left_join_cube");
 assertTrue(prjMgr.getRealizationsByTable("default", 
"default.test_kylin_fact").contains(cube));
 assertTrue(prjMgr.listAllRealizations("default").contains(cube));
 
@@ -162,7 +162,7 @@ public class ProjectManagerTest extends 
LocalFileMetadataTestCase {
 ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
 CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
 
-CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
+CubeInstance cube = cubeMgr.getCube("ci_left_join_cube");
 assertTrue(prjMgr.getRealizationsByTable("default", 
"default.test_kylin_fact").contains(cube));
 
 prjMgr.removeRealizationsFromProjects(RealizationType.CUBE, 
cube.getName());

http://git-wip-us.apache.org/repos/asf/kylin/blob/614b6ecb/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json 
b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
new file mode 100644
index

kylin git commit: minor, fix the typo

2016-12-19 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2293 f37537f18 -> aa7204343


minor, fix the typo


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa720434
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa720434
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa720434

Branch: refs/heads/KYLIN-2293
Commit: aa720434347fa3ab48a8581979084bf83c2b5bdb
Parents: f37537f
Author: Billy Liu 
Authored: Mon Dec 19 19:31:52 2016 +0800
Committer: Billy Liu 
Committed: Mon Dec 19 19:31:52 2016 +0800

--
 .../test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aa720434/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
 
b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index c052fea..a5ed6f8 100644
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -59,7 +59,7 @@ public class AppendTrieDictionaryTest extends 
LocalFileMetadataTestCase {
 
 @Before
 public void setUp() {
-staticCreateTegit stMetadata();
+staticCreateTestMetadata();
 System.setProperty("kylin.dictionary.append-entry-size", "5");
 System.setProperty("kylin.env.hdfs-working-dir", BASE_DIR);
 }

[02/15] kylin git commit: KYLIN-2283 A new data gen tool for CI

2016-12-19 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
--
diff --git a/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv 
b/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
deleted file mode 100644
index 07756c6..000
--- a/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
+++ /dev/null
@@ -1,402 +0,0 @@
-Auction,0,12,48027,0,10,Coins,2013-07-16 08:23:09,USER_Y,Coins & Paper 
Money,Paper Money: 
World,Asia,0,Ebay,USER_S,12,Merchant,2013-03-31,2013-03-31,1000157,2013-03-31,48027,1001,184.21,1
-Others,0,5,164261,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & 
Watches,Fashion 
Jewelry,Earrings,0,Ebay,USER_S,5,Occasional,2013-11-12,2013-11-10,1000158,2013-11-12,164261,1002,172.03,1
-Auction,15,14,82494,15,3,Books,2010-09-13 
07:05:54,USER_Y,BookMagazines,NULL,Comic 
Books,15,Australia,USER_S,14,Regular,2013-04-06,2013-03-31,1000161,2013-04-06,82494,1003,66.6,1
-Auction,15,12,66767,15,16,Home & Garden,2011-06-14 07:48:40,USER_Y,Home & 
Garden,NULL,Dogs,15,Australia,USER_S,12,Merchant,2013-05-17,2013-05-12,1000137,2013-05-17,66767,1004,92.98,1
-FP-GTC,0,5,152801,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & 
Watches,NULL,Earrings,0,Ebay,USER_S,5,Occasional,2013-05-20,2013-05-19,1000147,2013-05-20,152801,1005,132.33,1
-FP-GTC,0,13,43398,0,36,Home & Living,2013-02-20 23:11:43,USER_Y,Home & 
Garden,NULL,Cheese & 
Crackers,0,Ebay,USER_S,13,Entrepreneur,2013-06-16,2013-06-16,1000155,2013-06-16,43398,1006,7.12,1
-Auction,0,14,95173,0,37,Health & Beauty,2008-06-03 13:36:00,USER_Y,Health & 
Beauty,Bath & Body,Bath Sets & 
Kits,0,Ebay,USER_S,14,Regular,2013-06-14,2013-06-09,1000170,2013-06-14,95173,1007,204.28,1
-Auction,15,13,158666,15,28,Toys,2012-09-11 20:27:22,USER_Y,ToyHobbies,Action 
Figures,Anime & 
Manga,15,Australia,USER_S,13,Entrepreneur,2013-03-22,2013-03-17,1000173,2013-03-22,158666,1008,35.72,1
-Auction,0,12,12688,0,57,Sothebys,2008-06-03 13:36:00,USER_Y,eBay Premier,Books 
& Manuscripts,Books: 
Other,0,Ebay,USER_S,12,Merchant,2013-03-10,2013-03-10,1000178,2013-03-10,12688,1009,4.13,1
-FP-GTC,15,5,103324,15,9,Clothing & Accessories,2012-05-01 
08:54:26,USER_Y,ClothinShoeAccessories,Women's Shoes,Mixed 
Items,15,Australia,USER_S,5,Occasional,2013-11-01,2013-10-27,1000163,2013-11-01,103324,1010,27.48,1
-FP-GTC,15,14,108782,15,8,Auto Parts,2013-09-10 16:52:46,USER_Y,Vehicle Parts & 
Accessories,CaTruck Parts,Car Care & 
Cleaning,15,Australia,USER_S,14,Regular,2013-06-16,2013-06-16,1000166,2013-06-16,108782,1011,9.26,1
-Auction,0,12,80287,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets 
& Networking,Software,Office & 
Business,0,Ebay,USER_S,12,Merchant,2013-09-12,2013-09-08,1000167,2013-09-12,80287,1012,3.18,1
-Others,100,13,140746,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay 
Motors,Parts & Accessories,Vintage Car & Truck 
Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2013-09-28,2013-09-22,186,2013-09-28,140746,1013,3.18,1
-ABIN,0,14,87118,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting 
Goods,Outdoor 
Sports,Paintball,0,Ebay,USER_S,14,Regular,2013-06-15,2013-06-09,1000110,2013-06-15,87118,1014,377.94,1
-Auction,0,12,25147,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports 
MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Baseball-MLB,0,Ebay,USER_S,12,Merchant,2013-03-14,2013-03-10,1000113,2013-03-14,25147,1015,146.33,1
-FP-GTC,15,5,170302,15,15,Crafts,2010-09-13 
07:13:34,USER_Y,Crafts,Embroidery,Design 
CDs,15,Australia,USER_S,5,Occasional,2013-09-01,2013-09-01,169,2013-09-01,170302,1016,51.23,1
-FP-non GTC,0,13,53064,0,4,Business (Office & Industrial),2013-04-28 
20:37:19,USER_Y,Business & Industrial,Heavy Equipment,Antique & Vintage Farm 
Equip,0,Ebay,USER_S,13,Entrepreneur,2013-05-29,2013-05-26,179,2013-05-29,53064,1017,72.65,1
-Auction,0,13,132939,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & 
Watches,Fashion 
Jewelry,Other,0,Ebay,USER_S,13,Entrepreneur,2013-05-31,2013-05-26,180,2013-05-31,132939,1018,66.6,1
-Auction,15,12,113593,15,40,Cell Phones & Portable Electro,2012-09-11 
20:26:04,USER_Y,Phones,Mobile Phones,Mobile 
Phones,15,Australia,USER_S,12,Merchant,2013-03-18,2013-03-17,1000130,2013-03-18,113593,1019,9.26,1
-Auction,100,14,34273,100,8,Auto - Parts,2008-06-03 13:36:00,USER_Y,eBay 
Motors,Parts & 
Accessories,Motorcycle,100,Ebaymotors,USER_S,14,Regular,2013-07-19,2013-07-14,1000268,2013-07-19,34273,1020,583.44,1
-FP-GTC,15,14,106340,15,16,Home & Garden,2012-09-11 20:26:04,USER_Y,Home & 
Garden,Gardening,Hand 
Tools,15,Australia,USER_S,14,Regular,2013-06-23,2013-06-23,1000132,2013-06-23,106340,1021,638.72,1
-FP-GTC,15,14,150265,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby

[05/15] kylin git commit: KYLIN-2288 recogonize measure empty string as zero

2016-12-19 Thread billyliu

KYLIN-2288 recogonize measure empty string as zero

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d53ce9a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d53ce9a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d53ce9a

Branch: refs/heads/KYLIN-2293
Commit: 1d53ce9a0183ec369a681dec29b412f77f681f3e
Parents: d1175d2
Author: NiChuanlei 
Authored: Fri Dec 16 16:10:37 2016 +0800
Committer: Li Yang 
Committed: Mon Dec 19 11:50:14 2016 +0800

--
 .../java/org/apache/kylin/measure/basic/BigDecimalIngester.java| 2 +-
 .../main/java/org/apache/kylin/measure/basic/DoubleIngester.java   | 2 +-
 .../src/main/java/org/apache/kylin/measure/basic/LongIngester.java | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1d53ce9a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
index b51917c..c7541ab 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
@@ -33,7 +33,7 @@ public class BigDecimalIngester extends 
MeasureIngester {
 if (values.length > 1)
 throw new IllegalArgumentException();
 
-if (values[0] == null)
+if (values[0] == null || values[0].length() == 0)
 return new BigDecimal(0);
 else
 return new BigDecimal(values[0]);

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d53ce9a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
index 8356faa..a2f3980 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
@@ -37,7 +37,7 @@ public class DoubleIngester extends 
MeasureIngester {
 throw new IllegalArgumentException();
 
 DoubleMutable l = current;
-if (values[0] == null)
+if (values[0] == null || values[0].length() == 0)
 l.set(0L);
 else
 l.set(Double.parseDouble(values[0]));

http://git-wip-us.apache.org/repos/asf/kylin/blob/1d53ce9a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
index bfe6fe8..45a1634 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
@@ -37,7 +37,7 @@ public class LongIngester extends 
MeasureIngester {
 throw new IllegalArgumentException();
 
 LongMutable l = current;
-if (values[0] == null)
+if (values[0] == null || values[0].length() == 0)
 l.set(0L);
 else
 l.set(Long.parseLong(values[0]));

[07/15] kylin git commit: KYLIN-2297, Manually edit cube segment start/end time will throw error in UI

2016-12-19 Thread billyliu

KYLIN-2297, Manually edit cube segment start/end time will throw error in UI

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5fd3c564
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5fd3c564
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5fd3c564

Branch: refs/heads/KYLIN-2293
Commit: 5fd3c56408050b1bb2a939bbb7b2105b4806aef8
Parents: 7446c56
Author: luguosheng <550175...@qq.com>
Authored: Mon Dec 19 11:33:51 2016 +0800
Committer: zhongjian 
Committed: Mon Dec 19 13:54:36 2016 +0800

--
 webapp/app/js/directives/directives.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5fd3c564/webapp/app/js/directives/directives.js
--
diff --git a/webapp/app/js/directives/directives.js 
b/webapp/app/js/directives/directives.js
index f5051e8..bf1690c 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -278,10 +278,11 @@ KylinApp.directive('kylinPagination', function ($parse, 
$q) {
   });
 
   ctrl.$parsers.push(function (value) {
-if (isNaN(value)||value==null) {
+if(/\d{4}-\d{1,2}-\d{1,2}\s+(\d{1,2}:\d{1,2}:\d{1,2})/.test(value)) {
+  value=new Date(value);
+}else{
   return value;
 }
-//value = new Date(value.getFullYear(), value.getMonth(), 
value.getDate(), 0, 0, 0, 0);
 return value.getTime()-(6 * value.getTimezoneOffset());
   });
 }

[08/15] kylin git commit: KYLIN-2298, timer component get wrong seconds

2016-12-19 Thread billyliu

KYLIN-2298,timer component get wrong seconds

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/699a88eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/699a88eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/699a88eb

Branch: refs/heads/KYLIN-2293
Commit: 699a88eba6833449defa166cc3397a96ff765961
Parents: 5fd3c56
Author: luguosheng <550175...@qq.com>
Authored: Mon Dec 19 13:55:30 2016 +0800
Committer: zhongjian 
Committed: Mon Dec 19 13:57:22 2016 +0800

--
 webapp/app/js/directives/directives.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/699a88eb/webapp/app/js/directives/directives.js
--
diff --git a/webapp/app/js/directives/directives.js 
b/webapp/app/js/directives/directives.js
index bf1690c..3d18451 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -271,7 +271,7 @@ KylinApp.directive('kylinPagination', function ($parse, $q) 
{
 
 var hour = 
newDate.getHours()<10?'0'+newDate.getHours():newDate.getHours();
 var mins = 
newDate.getMinutes()<10?'0'+newDate.getMinutes():newDate.getMinutes();
-var seconds = 
newDate.getSeconds()<10?'0'+newDate.getSeconds():getSeconds();
+var seconds = 
newDate.getSeconds()<10?'0'+newDate.getSeconds():newDate.getSeconds();
 
 var viewVal = year+"-"+month+"-"+date+" "+hour+":"+mins+":"+seconds;
 return viewVal;

[14/15] kylin git commit: KYLIN-2269 Reduce MR memory usage for global dict

2016-12-19 Thread billyliu

KYLIN-2269 Reduce MR memory usage for global dict

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f50c0c87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f50c0c87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f50c0c87

Branch: refs/heads/KYLIN-2293
Commit: f50c0c87373e4abacd7106d527df4b885f0a88ea
Parents: 878107e
Author: kangkaisen 
Authored: Tue Dec 6 19:26:09 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 19 17:52:41 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 
 .../org/apache/kylin/cube/model/CubeDesc.java   | 22 
 .../cube/model/CubeJoinedFlatTableDesc.java | 15 -
 .../cube/model/CubeJoinedFlatTableEnrich.java   |  7 ++-
 .../org/apache/kylin/job/JoinedFlatTable.java   | 13 ++--
 .../metadata/model/IJoinedFlatTableDesc.java| 11 ++
 6 files changed, 60 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5153562..01d1d36 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -492,6 +492,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.source.hive.beeline-params", "");
 }
 
+public String getFlatHiveTableClusterByDictColumn() {
+return 
getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
+}
+
 @Deprecated
 public String getCreateFlatHiveTableMethod() {
 return getOptional("kylin.source.hive.create-flat-table-method", "1");

http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f6b68af..3b8d034 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -1090,6 +1090,28 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 return null;
 }
+
+/** Get a column which can be used to cluster the source table.
+ * To reduce memory footprint in base cuboid for global dict */
+// TODO handle more than one ultra high cardinality columns use global 
dict in one cube
+TblColRef getClusteredByColumn() {
+if (getDistributedByColumn() != null) {
+return null;
+}
+
+if (dictionaries == null) {
+return null;
+}
+
+String clusterByColumn = config.getFlatHiveTableClusterByDictColumn();
+for (DictionaryDesc dictDesc : dictionaries) {
+if 
(dictDesc.getColumnRef().getName().equalsIgnoreCase(clusterByColumn)) {
+return dictDesc.getColumnRef();
+}
+}
+
+return null;
+}
 
 public String getDictionaryBuilderClass(TblColRef col) {
 if (dictionaries == null)

http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index f37f86e..94e1a7c 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -49,11 +49,11 @@ public class CubeJoinedFlatTableDesc implements 
IJoinedFlatTableDesc {
 public CubeJoinedFlatTableDesc(CubeDesc cubeDesc) {
 this(cubeDesc, null);
 }
-
+
 public CubeJoinedFlatTableDesc(CubeSegment cubeSegment) {
 this(cubeSegment.getCubeDesc(), cubeSegment);
 }
-
+
 private CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment 
/* can be null */) {
 this.cubeDesc = cubeDesc;
 this.cubeSegment = cubeSegment;
@@ -68,7 +68,7 @@ public class CubeJoinedFlatTableDesc implements 
IJoinedFlatTableDesc {
 return "kylin_intermediate_" +

[13/15] kylin git commit: KYLIN 1875 minor, update model ang cube designer

2016-12-19 Thread billyliu

KYLIN 1875 minor,update model ang cube designer

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/878107e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/878107e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/878107e3

Branch: refs/heads/KYLIN-2293
Commit: 878107e35bf4ac3a7963f22ac00a805eea9c323f
Parents: 6f563df
Author: chenzhx <346839...@qq.com>
Authored: Mon Dec 19 14:16:47 2016 +0800
Committer: zhongjian 
Committed: Mon Dec 19 16:29:42 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 31 +
 webapp/app/js/controllers/cubeDimensions.js | 38 ---
 webapp/app/js/controllers/cubeEdit.js   |  6 +-
 webapp/app/js/controllers/cubeMeasures.js   | 12 ++--
 webapp/app/js/controllers/cubeSchema.js |  1 -
 webapp/app/js/controllers/modelDataModel.js |  6 --
 webapp/app/js/controllers/modelEdit.js  |  2 +-
 webapp/app/js/filters/filter.js | 17 -
 .../cubeDesigner/advanced_settings.html | 17 +
 .../app/partials/cubeDesigner/dimensions.html   | 69 ++--
 webapp/app/partials/cubeDesigner/measures.html  | 19 +++---
 .../app/partials/modelDesigner/data_model.html  | 34 --
 .../partials/modelDesigner/model_measures.html  |  2 +-
 13 files changed, 100 insertions(+), 154 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 20af908..760133a 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -21,7 +21,7 @@
 KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, 
$modal,cubeConfig,MetaModel,cubesManager,CubeDescModel,SweetAlert) {
   $scope.cubesManager = cubesManager;
   $scope.getTypeVersion=function(typename){
-var searchResult=/\[v(\d+)\]/.exec(typename);
+var searchResult=/\s*\(v(\d+)\)/.exec(typename);
 if(searchResult&){
   return searchResult.length&[1]||1;
 }else{
@@ -30,7 +30,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
   }
   $scope.removeVersion=function(typename){
 if(typename){
-  return typename.replace(/\[v\d+\]/g,"");
+  return typename.replace(/\s*\(v\d+\)/g,"");
 }
 return "";
   }
@@ -49,7 +49,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 _encoding=baseKey;
 var rowkeyObj = {
   column:item.column,
-  
encoding:_encoding+(item.encoding_version?"[v"+item.encoding_version+"]":"[v1]"),
+  encoding:_encoding+(item.encoding_version?"  
(v"+item.encoding_version+")":"  (v1)"),
   valueLength:_valueLength,
   isShardBy:item.isShardBy,
   encoding_version:item.encoding_version||1
@@ -109,31 +109,6 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 }
   }
 
-  $scope.removeRowkey = function(arr,index,item){
-if (index > -1) {
-  arr.splice(index, 1);
-}
-$scope.cubeMetaFrame.rowkey.rowkey_columns.splice(index,1);
-  }
-
-
-  $scope.addNewRowkeyColumn = function () {
-var rowkeyObj = {
-  column:"",
-  encoding:"dict",
-  valueLength:0,
-  isShardBy:"false"
-}
-
-$scope.convertedRowkeys.push(rowkeyObj);
-$scope.cubeMetaFrame.rowkey.rowkey_columns.push({
-  column:'',
-  encoding:'dict',
-  isShardBy:'false'
-});
-
-  };
-
   $scope.addNewHierarchy = function(grp){
 grp.select_rule.hierarchy_dims.push([]);
   }

http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeDimensions.js
--
diff --git a/webapp/app/js/controllers/cubeDimensions.js 
b/webapp/app/js/controllers/cubeDimensions.js
index e2787a5..80d54bb 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -206,9 +206,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 });
 
 modalInstance.result.then(function () {
-if (!$scope.dimState.editing) {
-$scope.doneAddDim();
-} else {
+if ($scope.dimState.editing) {
 $scope.doneEditDim();
 }
 
@@ -268,31 +266,31 @@ KylinApp.controller('CubeDimensionsCtrl', function 
($scope, $modal,MetaModel,cub
 
 };
 
-$scope.addDim = function (dimType) {
-$scope.newDimension = Dimension('', [], dimType);
-
-

[10/15] kylin git commit: KYLIN-2292 workaround for CALCITE-1540

2016-12-19 Thread billyliu

KYLIN-2292 workaround for CALCITE-1540


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ae4333c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ae4333c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ae4333c

Branch: refs/heads/KYLIN-2293
Commit: 4ae4333c82243c21d253008c9f5146f1e18f6e84
Parents: 6f9bd4a
Author: Hongbin Ma 
Authored: Fri Dec 16 17:21:37 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 19 14:16:08 2016 +0800

--
 .../adapter/enumerable/EnumerableWindow.java| 978 +++
 .../calcite/adapter/enumerable/PhysType.java| 209 
 .../adapter/enumerable/PhysTypeImpl.java| 654 +
 .../test/resources/query/sql_window/query11.sql |  23 +
 .../test/resources/query/sql_window/query12.sql |  26 +
 5 files changed, 1890 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae4333c/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
new file mode 100644
index 000..203ce02
--- /dev/null
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
@@ -0,0 +1,978 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.enumerable;
+
+import org.apache.calcite.adapter.enumerable.impl.WinAggAddContextImpl;
+import org.apache.calcite.adapter.enumerable.impl.WinAggResetContextImpl;
+import org.apache.calcite.adapter.enumerable.impl.WinAggResultContextImpl;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.linq4j.tree.BinaryExpression;
+import org.apache.calcite.linq4j.tree.BlockBuilder;
+import org.apache.calcite.linq4j.tree.BlockStatement;
+import org.apache.calcite.linq4j.tree.DeclarationStatement;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.ParameterExpression;
+import org.apache.calcite.linq4j.tree.Primitive;
+import org.apache.calcite.linq4j.tree.Statement;
+import org.apache.calcite.linq4j.tree.Types;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.CalcitePrepareImpl;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Window;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexWindowBound;
+import org.apache.calcite.runtime.SortedMultiMap;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.util.BuiltInMethod;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+/*
+ * OVERRIDE POINT: patching CALCITE-1540 on calcite 1.8.0
+ */
+
+/** Implementation of {@link org.apache.calcite.rel.core.Window} in
+ * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention 
enumerable calling convention}. */
+public class EnumerableWindow extends Window implements EnumerableRel {
+/** Creates an EnumerableWindowRel. */
+EnumerableWindow(RelOptCluster cluster, RelTraitSet

[11/15] kylin git commit: KYLIN-2295 Refactor CI, blend view cubes into the rest

2016-12-19 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
--
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json 
b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
index 655d4e7..d186f76 100644
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
@@ -1,408 +1,408 @@
-{
-  "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- 
-  "name" : "KYLIN_CAL_DT",
-  "columns" : [ {
-"id" : "1",
-"name" : "CAL_DT",
-"datatype" : "date"
-  }, {
-"id" : "2",
-"name" : "YEAR_BEG_DT",
-"datatype" : "date"
-  }, {
-"id" : "3",
-"name" : "QTR_BEG_DT",
-"datatype" : "date"
-  }, {
-"id" : "4",
-"name" : "MONTH_BEG_DT",
-"datatype" : "date"
-  }, {
-"id" : "5",
-"name" : "WEEK_BEG_DT",
-"datatype" : "date"
-  }, {
-"id" : "6",
-"name" : "AGE_FOR_YEAR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "7",
-"name" : "AGE_FOR_QTR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "8",
-"name" : "AGE_FOR_MONTH_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "9",
-"name" : "AGE_FOR_WEEK_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "10",
-"name" : "AGE_FOR_DT_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "11",
-"name" : "AGE_FOR_RTL_YEAR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "12",
-"name" : "AGE_FOR_RTL_QTR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "13",
-"name" : "AGE_FOR_RTL_MONTH_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "14",
-"name" : "AGE_FOR_RTL_WEEK_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "15",
-"name" : "AGE_FOR_CS_WEEK_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "16",
-"name" : "DAY_OF_CAL_ID",
-"datatype" : "int"
-  }, {
-"id" : "17",
-"name" : "DAY_OF_YEAR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "18",
-"name" : "DAY_OF_QTR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "19",
-"name" : "DAY_OF_MONTH_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "20",
-"name" : "DAY_OF_WEEK_ID",
-"datatype" : "int"
-  }, {
-"id" : "21",
-"name" : "WEEK_OF_YEAR_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "22",
-"name" : "WEEK_OF_CAL_ID",
-"datatype" : "int"
-  }, {
-"id" : "23",
-"name" : "MONTH_OF_QTR_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "24",
-"name" : "MONTH_OF_YEAR_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "25",
-"name" : "MONTH_OF_CAL_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "26",
-"name" : "QTR_OF_YEAR_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "27",
-"name" : "QTR_OF_CAL_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "28",
-"name" : "YEAR_OF_CAL_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "29",
-"name" : "YEAR_END_DT",
-"datatype" : "string"
-  }, {
-"id" : "30",
-"name" : "QTR_END_DT",
-"datatype" : "string"
-  }, {
-"id" : "31",
-"name" : "MONTH_END_DT",
-"datatype" : "string"
-  }, {
-"id" : "32",
-"name" : "WEEK_END_DT",
-"datatype" : "string"
-  }, {
-"id" : "33",
-"name" : "CAL_DT_NAME",
-"datatype" : "string"
-  }, {
-"id" : "34",
-"name" : "CAL_DT_DESC",
-"datatype" : "string"
-  }, {
-"id" : "35",
-"name" : "CAL_DT_SHORT_NAME",
-"datatype" : "string"
-  }, {
-"id" : "36",
-"name" : "YTD_YN_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "37",
-"name" : "QTD_YN_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "38",
-"name" : "MTD_YN_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "39",
-"name" : "WTD_YN_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "40",
-"name" : "SEASON_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "41",
-"name" : "DAY_IN_YEAR_COUNT",
-"datatype" : "smallint"
-  }, {
-"id" : "42",
-"name" : "DAY_IN_QTR_COUNT",
-"datatype" : "tinyint"
-  }, {
-"id" : "43",
-"name" : "DAY_IN_MONTH_COUNT",
-"datatype" : "tinyint"
-  }, {
-"id" : "44",
-"name" : "DAY_IN_WEEK_COUNT",
-"datatype" : "tinyint"
-  }, {
-"id" : "45",
-"name" : "RTL_YEAR_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "46",
-"name" : "RTL_QTR_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "47",
-"name" : "RTL_MONTH_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "48",
-"name" : "RTL_WEEK_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "49",
-"name" : "CS_WEEK_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "50",
-"name" : "CAL_DATE",
-"datatype" : "string"
-  }, {
-"id" : "51",
-"name" : "DAY_OF_WEEK",
-"datatype" : "string"
-  }, {
-"id" : "52",

[03/15] kylin git commit: KYLIN-2283 A new data gen tool for CI

2016-12-19 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java 
b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
new file mode 100644
index 000..82455ab
--- /dev/null
+++ 
b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.datagen;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DataGenTest extends LocalFileMetadataTestCase {
+
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
+
+@Test
+public void testCIConfigured() throws IOException {
+DataModelDesc model = getModel("test_kylin_inner_join_model_desc");
+ModelDataGenerator gen = new ModelDataGenerator(model, 100);
+gen.outprint = true;
+
+gen.generate();
+}
+
+@Test
+public void testSSBNoConfig() throws IOException {
+DataModelDesc model = getModel("ssb");
+ModelDataGenerator gen = new ModelDataGenerator(model, 100);
+gen.outprint = true;
+
+gen.generate();
+}
+
+private DataModelDesc getModel(String name) {
+MetadataManager mgr = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+DataModelDesc model = mgr.getDataModelDesc(name);
+return model;
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/examples/test_case_data/localmeta/data/data_gen_config.json
--
diff --git a/examples/test_case_data/localmeta/data/data_gen_config.json 
b/examples/test_case_data/localmeta/data/data_gen_config.json
deleted file mode 100644
index 15b3fd0..000
--- a/examples/test_case_data/localmeta/data/data_gen_config.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
-  "differentiateBoundary": "2013-07-01",
-  "columnConfigs": [
-{
-  "columnName": "lstg_format_name",
-  "valueSet": [
-"FP-GTC",
-"FP-non GTC",
-"ABIN",
-"Auction",
-"Others"
-  ],
-  "exclusive": true,
-  "differentiateByDateBoundary": true
-},
-{
-  "columnName": "BUYER_COUNTRY",
-  "valueSet": [
-"CN",
-"DE",
-"FR",
-"JP",
-"UK",
-"US"
-  ],
-  "exclusive": true
-},
-{
-  "columnName": "SELLER_COUNTRY",
-  "valueSet": [
-"CN",
-"DE",
-"FR",
-"JP",
-"UK",
-"US"
-  ],
-  "exclusive": true
-},
-{
-  "columnName": "SELLER_ID",
-  "valueSet": [
-"1000",
-"10001000"
-  ],
-  "asRange": true
-},
-{
-  "columnName": "ITEM_COUNT",
-  "valueSet": [
-"0",
-"200"
-  ],
-  "asRange": true
-},
-{
-  "columnName": "PRICE",
-  "valueSet": [
-"0",
-"1000"
-  ],
-  "asRange": true
-}
-  ]
-}

[01/15] kylin git commit: KYLIN-2283 A new data gen tool for CI [Forced Update!]

2016-12-19 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2293 ef76d4800 -> f37537f18 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index 2f8991b..df6eb2f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -217,7 +217,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase 
{
 
 PreparedStatement statement = conn.prepareStatement("select 
LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact 
" + "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
 
-statement.setString(1, "FP-GTC_A");
+statement.setString(1, "FP-GTC");
 
 ResultSet rs = statement.executeQuery();
 
@@ -225,7 +225,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase 
{
 
 String format_name = rs.getString(1);
 
-Assert.assertTrue("FP-GTC_A".equals(format_name));
+Assert.assertTrue("FP-GTC".equals(format_name));
 
 rs.close();
 statement.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/kylin-it/src/test/resources/query/sql_timeout/query02.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_timeout/query02.sql 
b/kylin-it/src/test/resources/query/sql_timeout/query02.sql
index 2f187a4..a00096c 100644
--- a/kylin-it/src/test/resources/query/sql_timeout/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_timeout/query02.sql
@@ -16,4 +16,4 @@
 -- limitations under the License.
 --
 
-select seller_id,lstg_format_name,sum(price) from test_kylin_fact group by 
seller_id,lstg_format_name
\ No newline at end of file
+select seller_id,cal_dt,lstg_format_name,sum(price) from test_kylin_fact group 
by seller_id,cal_dt,lstg_format_name
\ No newline at end of file

[04/15] kylin git commit: KYLIN-2283 A new data gen tool for CI

2016-12-19 Thread billyliu

KYLIN-2283 A new data gen tool for CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d1175d2c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d1175d2c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d1175d2c

Branch: refs/heads/KYLIN-2293
Commit: d1175d2c414853f4d223601a2db096f7f963ce72
Parents: e14f4e1
Author: Li Yang 
Authored: Thu Dec 15 18:51:00 2016 +0800
Committer: Yang Li 
Committed: Sun Dec 18 17:23:18 2016 +0800

--
 .../java/org/apache/kylin/job/DataGenTest.java  |  56 --
 .../java/org/apache/kylin/job/DeployUtil.java   |  27 +-
 .../apache/kylin/job/dataGen/ColumnConfig.java  |  80 ---
 .../kylin/job/dataGen/FactTableGenerator.java   | 696 ---
 .../org/apache/kylin/job/dataGen/GenConfig.java |  92 ---
 .../apache/kylin/metadata/model/ColumnDesc.java |   8 +
 .../apache/kylin/metadata/model/TableDesc.java  |   9 +
 .../kylin/source/datagen/ColumnGenConfig.java   | 110 +++
 .../kylin/source/datagen/ColumnGenerator.java   | 361 ++
 .../source/datagen/ModelDataGenerator.java  | 282 
 .../kylin/source/datagen/TableGenConfig.java|  48 ++
 .../org/apache/kylin/source/datagen/Util.java   |  75 ++
 .../topn/TopNCounterSerializerTest.java |  76 --
 .../hllc/NewHyperLogLogBenchmarkTest.java   |   2 +
 .../measure/topn/TopNCounterSerializerTest.java |  76 ++
 .../kylin/source/datagen/DataGenTest.java   |  67 ++
 .../localmeta/data/data_gen_config.json |  65 --
 .../localmeta/data/flatten_data_for_ii.csv  | 402 ---
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |  32 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |   4 +-
 .../resources/query/sql_timeout/query02.sql |   2 +-
 21 files changed, 1070 insertions(+), 1500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/assembly/src/test/java/org/apache/kylin/job/DataGenTest.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DataGenTest.java 
b/assembly/src/test/java/org/apache/kylin/job/DataGenTest.java
deleted file mode 100644
index af4f9fb..000
--- a/assembly/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.job.dataGen.FactTableGenerator;
-import org.apache.kylin.metadata.MetadataManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- *
- */
-public class DataGenTest extends LocalFileMetadataTestCase {
-
-@Before
-public void before() throws Exception {
-this.createTestMetadata();
-MetadataManager.clearCache();
-}
-
-@After
-public void after() throws Exception {
-this.cleanupTestMetadata();
-}
-
-@Test
-public void testBasics() throws Exception {
-String content = 
FactTableGenerator.generate("test_kylin_cube_with_slr_ready", "1", "1", 
null);// default  settings
-//System.out.println(content);
-assertTrue(content.contains("FP-non GTC"));
-assertTrue(content.contains("ABIN"));
-
-//DeployUtil.overrideFactTableData(content, "default.test_kylin_fact");
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 23b3670..8fc583d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -33,20 +33,21 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import

[15/15] kylin git commit: KYLIN-2293 remove the hack set methods in KylinConfig

2016-12-19 Thread billyliu

KYLIN-2293 remove the hack set methods in KylinConfig


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f37537f1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f37537f1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f37537f1

Branch: refs/heads/KYLIN-2293
Commit: f37537f18c656efb8f4dbc933306bd04d6cefa00
Parents: f50c0c8
Author: Billy Liu 
Authored: Mon Dec 19 19:23:12 2016 +0800
Committer: Billy Liu 
Committed: Mon Dec 19 19:23:12 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 39 
 .../kylin/common/util/MailServiceTest.java  |  2 +-
 .../org/apache/kylin/cube/CubeManagerTest.java  |  4 +-
 .../kylin/dict/AppendTrieDictionaryTest.java| 30 ++-
 .../kylin/provision/BuildCubeWithEngine.java|  4 +-
 5 files changed, 17 insertions(+), 62 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f37537f1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 01d1d36..dd6b77c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -240,11 +240,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.dictionary.append-version-ttl", 
"25920"));
 }
 
-// for test
-public void setAppendDictEntrySize(int entrySize) {
-setProperty("kylin.dictionary.append-entry-size", 
String.valueOf(entrySize));
-}
-
 public int getCachedSnapshotMaxEntrySize() {
 return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", 
"500"));
 }
@@ -298,10 +293,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
 }
 
-public void setMaxBuildingSegments(int maxBuildingSegments) {
-setProperty("kylin.cube.max-building-segments", 
String.valueOf(maxBuildingSegments));
-}
-
 // 

 // JOB
 // 

@@ -322,10 +313,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.parseBoolean(getOptional("kylin.job.use-remote-cli"));
 }
 
-public void setRunAsRemoteCommand(String v) {
-setProperty("kylin.job.use-remote-cli", v);
-}
-
 public int getRemoteHadoopCliPort() {
 return Integer.parseInt(getOptional("kylin.job.remote-cli-port", 
"22"));
 }
@@ -334,26 +321,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.job.remote-cli-hostname");
 }
 
-public void setRemoteHadoopCliHostname(String v) {
-setProperty("kylin.job.remote-cli-hostname", v);
-}
-
 public String getRemoteHadoopCliUsername() {
 return getOptional("kylin.job.remote-cli-username");
 }
 
-public void setRemoteHadoopCliUsername(String v) {
-setProperty("kylin.job.remote-cli-username", v);
-}
-
 public String getRemoteHadoopCliPassword() {
 return getOptional("kylin.job.remote-cli-password");
 }
 
-public void setRemoteHadoopCliPassword(String v) {
-setProperty("kylin.job.remote-cli-password", v);
-}
-
 public String getCliWorkingDir() {
 return getOptional("kylin.job.remote-cli-working-dir");
 }
@@ -381,10 +356,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.job.notification-enabled", "false"));
 }
 
-public void setMailEnabled(boolean enable) {
-setProperty("kylin.job.notification-enabled", "" + enable);
-}
-
 public String getMailHost() {
 return getOptional("kylin.job.notification-mail-host", "");
 }
@@ -522,11 +493,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.storage.url");
 }
 
-// for test only
-public void setStorageUrl(String storageUrl) {
-setProperty("kylin.storage.url", storageUrl);
-}
-
 public String getHBaseClusterFs() {
 return getOptional("kylin.storage.hbase.cluster-fs", "");
 }
@@ -582,11 +548,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.storage.hbase.max-region-count", "500"));

[06/15] kylin git commit: minor, add equals and hashcode to RowKeyColDesc

2016-12-19 Thread billyliu

minor, add equals and hashcode to RowKeyColDesc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7446c56b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7446c56b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7446c56b

Branch: refs/heads/KYLIN-2293
Commit: 7446c56b1aca87fce9868c9e1f0674299b05c405
Parents: 1d53ce9
Author: lidongsjtu 
Authored: Sat Dec 17 13:46:22 2016 +0800
Committer: lidongsjtu 
Committed: Mon Dec 19 12:57:14 2016 +0800

--
 .../apache/kylin/cube/model/RowKeyColDesc.java  | 26 
 1 file changed, 26 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7446c56b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 3b49323..ef34a9b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -155,8 +155,34 @@ public class RowKeyColDesc {
 }
 
 @Override
+public int hashCode() {
+final int prime = 31;
+int result = 1;
+result = prime * result + ((column == null) ? 0 : column.hashCode());
+return result;
+}
+
+@Override
 public String toString() {
 return Objects.toStringHelper(this).add("column", 
column).add("encoding", encoding).toString();
 }
 
+@Override
+public boolean equals(Object o) {
+if (this == o) {
+return true;
+}
+
+if (o == null || getClass() != o.getClass()) {
+return false;
+}
+
+RowKeyColDesc that = (RowKeyColDesc) o;
+
+if (column != null ? !column.equals(that.column) : that.column != 
null) {
+return false;
+}
+
+return true;
+}
 }
\ No newline at end of file

[09/15] kylin git commit: KYLIN-2290 minor improvements on limit

2016-12-19 Thread billyliu

KYLIN-2290 minor improvements on limit

KYLIN-2290 fix compile

KYLIN-2290 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f9bd4a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f9bd4a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f9bd4a9

Branch: refs/heads/KYLIN-2293
Commit: 6f9bd4a9998cff24c97bce0e6d86282f48fef388
Parents: 699a88e
Author: Hongbin Ma 
Authored: Fri Dec 16 16:28:18 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 19 14:16:00 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  1 +
 .../kylin/cube/CubeCapabilityChecker.java   |  5 ++
 .../kylin/gridtable/GTScanRequestBuilder.java   |  2 +-
 .../kylin/metadata/realization/SQLDigest.java   |  4 +-
 .../apache/kylin/storage/StorageContext.java| 74 +++-
 .../storage/gtrecord/CubeScanRangePlanner.java  |  9 +--
 .../gtrecord/GTCubeStorageQueryBase.java|  1 -
 .../gtrecord/SequentialCubeTupleIterator.java   |  6 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  1 -
 .../org/apache/kylin/query/KylinTestBase.java   |  7 +-
 .../kylin/storage/hbase/ITStorageTest.java  |  2 +-
 .../kylin/query/relnode/OLAPAggregateRel.java   |  4 ++
 .../apache/kylin/query/relnode/OLAPContext.java |  6 +-
 .../kylin/query/relnode/OLAPFilterRel.java  |  2 +-
 .../kylin/query/relnode/OLAPLimitRel.java   |  6 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  7 +-
 16 files changed, 60 insertions(+), 77 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 610c2af..5153562 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -778,6 +778,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
 }
 
+@Deprecated//Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 38faed9..c45144b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -120,6 +120,11 @@ public class CubeCapabilityChecker {
 return result;
 }
 
+if (digest.limitPrecedesAggr) {
+logger.info("Exclude cube " + cube.getName() + " because there's 
limit preceding aggregation");
+return result;
+}
+
 if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) {
 result.influences.add(new CapabilityInfluence() {
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index f542de1..bcec1f4 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -36,7 +36,7 @@ public class GTScanRequestBuilder {
 private boolean allowStorageAggregation = true;
 private double aggCacheMemThreshold = 0;
 private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage 
should terminate itself when $storageScanRowNumThreshold cuboid rows are 
scanned, and throw exception.   
-private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit 
working when $toragePushDownLimit aggregated rows are produced. 
+private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit 
scanning safely when $toragePushDownLimit aggregated rows are produced. 
 private long startTime = -1;
 private long

kylin git commit: pass UT

2016-12-19 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2294 8dd1e2b4d -> 21531d734


pass UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21531d73
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21531d73
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21531d73

Branch: refs/heads/KYLIN-2294
Commit: 21531d734134f95d4313888ced38467570932e23
Parents: 8dd1e2b
Author: Li Yang 
Authored: Mon Dec 19 18:27:55 2016 +0800
Committer: Li Yang 
Committed: Mon Dec 19 18:27:55 2016 +0800

--
 .../java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java | 2 +-
 .../java/org/apache/kylin/measure/hllc/HLLCounterTest.java| 2 +-
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 4 ++--
 .../model_desc/test_kylin_inner_join_model_desc.json  | 7 +++
 .../localmeta/model_desc/test_kylin_left_join_model_desc.json | 7 +++
 5 files changed, 10 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/21531d73/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
index c4a97cd..427d9a7 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
@@ -157,7 +157,7 @@ public class HLLCounterOldTest {
 public void mergeTest() throws IOException {
 double error = 0;
 int n = 100;
-for (int i = 0; i < n; i++) {
+for (int i = 0; i < n; i += 10) {
 double e = merge(i);
 error += e;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/21531d73/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java 
b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
index 26ad4a7..cd75eac 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
@@ -82,7 +82,7 @@ public class HLLCounterTest {
 public void mergeTest() throws IOException {
 double error = 0;
 int n = 100;
-for (int i = 0; i < n; i++) {
+for (int i = 0; i < n; i += 10) {
 double e = merge(i);
 error += e;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/21531d73/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 9c26be9..9aeaa2e 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -121,7 +121,7 @@
   "expression" : "COUNT_DISTINCT",
   "parameter" : {
 "type" : "column",
-"value" : "USER_ID",
+"value" : "ORDER_ID",
 "next_parameter" : null
   },
   "returntype" : "bitmap"
@@ -210,7 +210,7 @@
 "dependent_measure_ref" : null
   } ],
   "dictionaries": [ {
-"column": "USER_ID",
+"column": "ORDER_ID",
 "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
   } ],
   "rowkey" : {

http://git-wip-us.apache.org/repos/asf/kylin/blob/21531d73/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
 
b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index 16ce8ae..49758dd 100644
--- 
a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ 
b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -63,7 +63,8 @@
 "TRANS_ID",
 "CAL_DT",
 "LEAF_CATEG_ID",
-"SELLER_ID"
+"SELLER_ID",
+"ORDER_ID"
   ]
 },
 {
@@ -105,9 +106,7 @@
   ],
   "metrics": [
 "PRICE",
-"ITEM_COUNT",
-"SELLER_ID",
-"USER_ID"
+"ITEM_COUNT"
   ],

[kylin] Git Push Summary

2016-12-19 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2295 [deleted] 1d3579d61

kylin git commit: add CI metadata, pending cube desc

2016-12-19 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2294 [created] 8dd1e2b4d


add CI metadata, pending cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8dd1e2b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8dd1e2b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8dd1e2b4

Branch: refs/heads/KYLIN-2294
Commit: 8dd1e2b4d3cc72028be8f9b121a6479ffa2bbc6d
Parents: 6f563df
Author: Li Yang 
Authored: Mon Dec 19 18:03:37 2016 +0800
Committer: Li Yang 
Committed: Mon Dec 19 18:03:37 2016 +0800

--
 .../localmeta/cube/ci_inner_join_cube.json  |  10 ++
 .../localmeta/cube/ci_left_join_cube.json   |  10 ++
 .../localmeta/hybrid/ci_inner_join_hybrid.json  |  14 ++
 .../hybrid/test_kylin_hybrid_inner_join.json|  14 --
 .../model_desc/ci_inner_join_model.json | 139 +++
 .../model_desc/ci_left_join_model.json  | 139 +++
 .../localmeta/project/default.json  |  26 ++--
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |  41 ++
 .../localmeta/table/DEFAULT.TEST_ORDER.json |  18 +++
 .../apache/kylin/query/ITCombinationTest.java   |  15 +-
 .../apache/kylin/query/ITKylinQueryTest.java|   3 -
 11 files changed, 358 insertions(+), 71 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8dd1e2b4/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json
--
diff --git a/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json 
b/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json
new file mode 100644
index 000..f055be0
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ci_inner_join_cube.json
@@ -0,0 +1,10 @@
+{
+  "uuid" : "8372c3b7-a33e-4b69-83dd-0bb8b1f8117e",
+  "last_modified" : 0,
+  "name" : "ci_inner_join_cube",
+  "owner" : null,
+  "descriptor" : "ci_inner_join_cube",
+  "segments" : [ ],
+  "status" : "DISABLED",
+  "create_time" : null
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/8dd1e2b4/examples/test_case_data/localmeta/cube/ci_left_join_cube.json
--
diff --git a/examples/test_case_data/localmeta/cube/ci_left_join_cube.json 
b/examples/test_case_data/localmeta/cube/ci_left_join_cube.json
new file mode 100644
index 000..c3fddf0
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/ci_left_join_cube.json
@@ -0,0 +1,10 @@
+{
+  "uuid" : "56e1a32a-a33e-4b69-83dd-0bb8b1f80354",
+  "last_modified" : 0,
+  "name" : "ci_left_join_cube",
+  "owner" : null,
+  "descriptor" : "ci_left_join_cube",
+  "segments" : [ ],
+  "status" : "DISABLED",
+  "create_time" : null
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/8dd1e2b4/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json
--
diff --git a/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json 
b/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json
new file mode 100644
index 000..05d30c5
--- /dev/null
+++ b/examples/test_case_data/localmeta/hybrid/ci_inner_join_hybrid.json
@@ -0,0 +1,14 @@
+{
+  "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
+ 
+  "name": "ci_inner_join_hybrid",
+  "realizations": [
+{
+  "type": "CUBE",
+  "realization": "ci_inner_join_cube"
+}
+  ],
+  "cost": 100,
+  "last_modified": 1420016227424,
+  "create_time": null
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/8dd1e2b4/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
--
diff --git 
a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json 
b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
deleted file mode 100644
index 443c0a3..000
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
- 
-  "name": "test_kylin_hybrid_inner_join",
-  "realizations": [
-{
-  "type": "CUBE",
-  "realization": "test_kylin_cube_without_slr_empty"
-}
-  ],
-  "cost": 100,
-  "last_modified": 1420016227424,
-  "create_time": null
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/8dd1e2b4/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
--
diff --git 
a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json

kylin git commit: KYLIN-2269 Reduce MR memory usage for global dict

2016-12-19 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 878107e35 -> f50c0c873


KYLIN-2269 Reduce MR memory usage for global dict

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f50c0c87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f50c0c87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f50c0c87

Branch: refs/heads/master
Commit: f50c0c87373e4abacd7106d527df4b885f0a88ea
Parents: 878107e
Author: kangkaisen 
Authored: Tue Dec 6 19:26:09 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 19 17:52:41 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 
 .../org/apache/kylin/cube/model/CubeDesc.java   | 22 
 .../cube/model/CubeJoinedFlatTableDesc.java | 15 -
 .../cube/model/CubeJoinedFlatTableEnrich.java   |  7 ++-
 .../org/apache/kylin/job/JoinedFlatTable.java   | 13 ++--
 .../metadata/model/IJoinedFlatTableDesc.java| 11 ++
 6 files changed, 60 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5153562..01d1d36 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -492,6 +492,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.source.hive.beeline-params", "");
 }
 
+public String getFlatHiveTableClusterByDictColumn() {
+return 
getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
+}
+
 @Deprecated
 public String getCreateFlatHiveTableMethod() {
 return getOptional("kylin.source.hive.create-flat-table-method", "1");

http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f6b68af..3b8d034 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -1090,6 +1090,28 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 return null;
 }
+
+/** Get a column which can be used to cluster the source table.
+ * To reduce memory footprint in base cuboid for global dict */
+// TODO handle more than one ultra high cardinality columns use global 
dict in one cube
+TblColRef getClusteredByColumn() {
+if (getDistributedByColumn() != null) {
+return null;
+}
+
+if (dictionaries == null) {
+return null;
+}
+
+String clusterByColumn = config.getFlatHiveTableClusterByDictColumn();
+for (DictionaryDesc dictDesc : dictionaries) {
+if 
(dictDesc.getColumnRef().getName().equalsIgnoreCase(clusterByColumn)) {
+return dictDesc.getColumnRef();
+}
+}
+
+return null;
+}
 
 public String getDictionaryBuilderClass(TblColRef col) {
 if (dictionaries == null)

http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index f37f86e..94e1a7c 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -49,11 +49,11 @@ public class CubeJoinedFlatTableDesc implements 
IJoinedFlatTableDesc {
 public CubeJoinedFlatTableDesc(CubeDesc cubeDesc) {
 this(cubeDesc, null);
 }
-
+
 public CubeJoinedFlatTableDesc(CubeSegment cubeSegment) {
 this(cubeSegment.getCubeDesc(), cubeSegment);
 }
-
+
 private CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment 
/* can be null */) {
 this.cubeDesc = cubeDesc;
 this.cubeSegment = cubeSegment;
@@ -68,7 +68,7 @@ public class CubeJoinedFlatTableDesc implements

kylin git commit: KYLIN 1875 minor,update model ang cube designer

2016-12-19 Thread zhongjian

Repository: kylin
Updated Branches:
  refs/heads/master 6f563df4a -> 878107e35


KYLIN 1875 minor,update model ang cube designer

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/878107e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/878107e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/878107e3

Branch: refs/heads/master
Commit: 878107e35bf4ac3a7963f22ac00a805eea9c323f
Parents: 6f563df
Author: chenzhx <346839...@qq.com>
Authored: Mon Dec 19 14:16:47 2016 +0800
Committer: zhongjian 
Committed: Mon Dec 19 16:29:42 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 31 +
 webapp/app/js/controllers/cubeDimensions.js | 38 ---
 webapp/app/js/controllers/cubeEdit.js   |  6 +-
 webapp/app/js/controllers/cubeMeasures.js   | 12 ++--
 webapp/app/js/controllers/cubeSchema.js |  1 -
 webapp/app/js/controllers/modelDataModel.js |  6 --
 webapp/app/js/controllers/modelEdit.js  |  2 +-
 webapp/app/js/filters/filter.js | 17 -
 .../cubeDesigner/advanced_settings.html | 17 +
 .../app/partials/cubeDesigner/dimensions.html   | 69 ++--
 webapp/app/partials/cubeDesigner/measures.html  | 19 +++---
 .../app/partials/modelDesigner/data_model.html  | 34 --
 .../partials/modelDesigner/model_measures.html  |  2 +-
 13 files changed, 100 insertions(+), 154 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 20af908..760133a 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -21,7 +21,7 @@
 KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, 
$modal,cubeConfig,MetaModel,cubesManager,CubeDescModel,SweetAlert) {
   $scope.cubesManager = cubesManager;
   $scope.getTypeVersion=function(typename){
-var searchResult=/\[v(\d+)\]/.exec(typename);
+var searchResult=/\s*\(v(\d+)\)/.exec(typename);
 if(searchResult&){
   return searchResult.length&[1]||1;
 }else{
@@ -30,7 +30,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
   }
   $scope.removeVersion=function(typename){
 if(typename){
-  return typename.replace(/\[v\d+\]/g,"");
+  return typename.replace(/\s*\(v\d+\)/g,"");
 }
 return "";
   }
@@ -49,7 +49,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 _encoding=baseKey;
 var rowkeyObj = {
   column:item.column,
-  
encoding:_encoding+(item.encoding_version?"[v"+item.encoding_version+"]":"[v1]"),
+  encoding:_encoding+(item.encoding_version?"  
(v"+item.encoding_version+")":"  (v1)"),
   valueLength:_valueLength,
   isShardBy:item.isShardBy,
   encoding_version:item.encoding_version||1
@@ -109,31 +109,6 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 }
   }
 
-  $scope.removeRowkey = function(arr,index,item){
-if (index > -1) {
-  arr.splice(index, 1);
-}
-$scope.cubeMetaFrame.rowkey.rowkey_columns.splice(index,1);
-  }
-
-
-  $scope.addNewRowkeyColumn = function () {
-var rowkeyObj = {
-  column:"",
-  encoding:"dict",
-  valueLength:0,
-  isShardBy:"false"
-}
-
-$scope.convertedRowkeys.push(rowkeyObj);
-$scope.cubeMetaFrame.rowkey.rowkey_columns.push({
-  column:'',
-  encoding:'dict',
-  isShardBy:'false'
-});
-
-  };
-
   $scope.addNewHierarchy = function(grp){
 grp.select_rule.hierarchy_dims.push([]);
   }

http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeDimensions.js
--
diff --git a/webapp/app/js/controllers/cubeDimensions.js 
b/webapp/app/js/controllers/cubeDimensions.js
index e2787a5..80d54bb 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -206,9 +206,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, 
$modal,MetaModel,cub
 });
 
 modalInstance.result.then(function () {
-if (!$scope.dimState.editing) {
-$scope.doneAddDim();
-} else {
+if ($scope.dimState.editing) {
 $scope.doneEditDim();
 }
 
@@ -268,31 +266,31 @@ KylinApp.controller('CubeDimensionsCtrl', function 
($scope, $modal,MetaModel,cub
 
 };
 
-$scope.addDim = function (dimType) {
-

kylin git commit: KYLIN-1770, remove DateRangeRules

2016-12-19 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1770 7bb4ffafc -> b111c601c


KYLIN-1770, remove DateRangeRules


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b111c601
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b111c601
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b111c601

Branch: refs/heads/KYLIN-1770
Commit: b111c601c147a7ed1259cf0ea33a0627c91c24c3
Parents: 7bb4ffa
Author: Billy Liu 
Authored: Mon Dec 19 16:22:33 2016 +0800
Committer: Billy Liu 
Committed: Mon Dec 19 16:22:33 2016 +0800

--
 .../main/java/org/apache/kylin/query/relnode/OLAPTableScan.java| 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b111c601/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index b157f34..452170a 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -45,6 +45,7 @@ import 
org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
 import org.apache.calcite.rel.rules.AggregateJoinTransposeRule;
 import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
 import org.apache.calcite.rel.rules.AggregateUnionTransposeRule;
+import org.apache.calcite.rel.rules.DateRangeRules;
 import org.apache.calcite.rel.rules.FilterJoinRule;
 import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
 import org.apache.calcite.rel.rules.JoinCommuteRule;
@@ -172,6 +173,7 @@ public class OLAPTableScan extends TableScan implements 
OLAPRel, EnumerableRel {
 planner.removeRule(JoinUnionTransposeRule.LEFT_UNION);
 planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
 planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
+planner.removeRule(DateRangeRules.FILTER_INSTANCE);
 // distinct count will be split into a separated query that is joined 
with the left query
 planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);

[1/2] kylin git commit: KYLIN-2295 Refactor CI, blend view cubes into the rest

2016-12-19 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 4ae4333c8 -> 6f563df4a


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
--
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json 
b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
index 655d4e7..d186f76 100644
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
@@ -1,408 +1,408 @@
-{
-  "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- 
-  "name" : "KYLIN_CAL_DT",
-  "columns" : [ {
-"id" : "1",
-"name" : "CAL_DT",
-"datatype" : "date"
-  }, {
-"id" : "2",
-"name" : "YEAR_BEG_DT",
-"datatype" : "date"
-  }, {
-"id" : "3",
-"name" : "QTR_BEG_DT",
-"datatype" : "date"
-  }, {
-"id" : "4",
-"name" : "MONTH_BEG_DT",
-"datatype" : "date"
-  }, {
-"id" : "5",
-"name" : "WEEK_BEG_DT",
-"datatype" : "date"
-  }, {
-"id" : "6",
-"name" : "AGE_FOR_YEAR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "7",
-"name" : "AGE_FOR_QTR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "8",
-"name" : "AGE_FOR_MONTH_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "9",
-"name" : "AGE_FOR_WEEK_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "10",
-"name" : "AGE_FOR_DT_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "11",
-"name" : "AGE_FOR_RTL_YEAR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "12",
-"name" : "AGE_FOR_RTL_QTR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "13",
-"name" : "AGE_FOR_RTL_MONTH_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "14",
-"name" : "AGE_FOR_RTL_WEEK_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "15",
-"name" : "AGE_FOR_CS_WEEK_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "16",
-"name" : "DAY_OF_CAL_ID",
-"datatype" : "int"
-  }, {
-"id" : "17",
-"name" : "DAY_OF_YEAR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "18",
-"name" : "DAY_OF_QTR_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "19",
-"name" : "DAY_OF_MONTH_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "20",
-"name" : "DAY_OF_WEEK_ID",
-"datatype" : "int"
-  }, {
-"id" : "21",
-"name" : "WEEK_OF_YEAR_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "22",
-"name" : "WEEK_OF_CAL_ID",
-"datatype" : "int"
-  }, {
-"id" : "23",
-"name" : "MONTH_OF_QTR_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "24",
-"name" : "MONTH_OF_YEAR_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "25",
-"name" : "MONTH_OF_CAL_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "26",
-"name" : "QTR_OF_YEAR_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "27",
-"name" : "QTR_OF_CAL_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "28",
-"name" : "YEAR_OF_CAL_ID",
-"datatype" : "smallint"
-  }, {
-"id" : "29",
-"name" : "YEAR_END_DT",
-"datatype" : "string"
-  }, {
-"id" : "30",
-"name" : "QTR_END_DT",
-"datatype" : "string"
-  }, {
-"id" : "31",
-"name" : "MONTH_END_DT",
-"datatype" : "string"
-  }, {
-"id" : "32",
-"name" : "WEEK_END_DT",
-"datatype" : "string"
-  }, {
-"id" : "33",
-"name" : "CAL_DT_NAME",
-"datatype" : "string"
-  }, {
-"id" : "34",
-"name" : "CAL_DT_DESC",
-"datatype" : "string"
-  }, {
-"id" : "35",
-"name" : "CAL_DT_SHORT_NAME",
-"datatype" : "string"
-  }, {
-"id" : "36",
-"name" : "YTD_YN_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "37",
-"name" : "QTD_YN_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "38",
-"name" : "MTD_YN_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "39",
-"name" : "WTD_YN_ID",
-"datatype" : "tinyint"
-  }, {
-"id" : "40",
-"name" : "SEASON_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "41",
-"name" : "DAY_IN_YEAR_COUNT",
-"datatype" : "smallint"
-  }, {
-"id" : "42",
-"name" : "DAY_IN_QTR_COUNT",
-"datatype" : "tinyint"
-  }, {
-"id" : "43",
-"name" : "DAY_IN_MONTH_COUNT",
-"datatype" : "tinyint"
-  }, {
-"id" : "44",
-"name" : "DAY_IN_WEEK_COUNT",
-"datatype" : "tinyint"
-  }, {
-"id" : "45",
-"name" : "RTL_YEAR_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "46",
-"name" : "RTL_QTR_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "47",
-"name" : "RTL_MONTH_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "48",
-"name" : "RTL_WEEK_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "49",
-"name" : "CS_WEEK_BEG_DT",
-"datatype" : "string"
-  }, {
-"id" : "50",
-"name" : "CAL_DATE",
-"datatype" : "string"
-  }, {
-"id" : "51",

[2/2] kylin git commit: KYLIN-2295 Refactor CI, blend view cubes into the rest

2016-12-19 Thread liyang

KYLIN-2295 Refactor CI, blend view cubes into the rest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f563df4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f563df4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f563df4

Branch: refs/heads/master
Commit: 6f563df4afa46af61ffcbbc3ad599098b737d07a
Parents: 4ae4333
Author: Yang Li 
Authored: Sun Dec 18 20:24:25 2016 +0800
Committer: Li Yang 
Committed: Mon Dec 19 16:03:22 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  22 +-
 .../kylin/source/datagen/ColumnGenConfig.java   |  18 +-
 .../source/datagen/ModelDataGenerator.java  |  10 +-
 .../kylin/source/datagen/TableGenConfig.java|   5 +-
 ...t_kylin_cube_with_view_inner_join_empty.json |  10 -
 ...st_kylin_cube_with_view_left_join_empty.json |  10 -
 ...st_kylin_cube_with_view_inner_join_desc.json | 169 
 ...est_kylin_cube_with_view_left_join_desc.json | 169 
 .../localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv |   8 -
 .../data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv |   8 +
 .../test_kylin_inner_join_view_model_desc.json  | 119 ---
 .../test_kylin_left_join_view_model_desc.json   | 119 ---
 .../localmeta/project/default.json  |  14 +-
 .../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json  | 302 +++
 .../table/DEFAULT.TEST_KYLIN_COUNTRY.json   |  46 +-
 .../table/DEFAULT.TEST_KYLIN_FACT.json  | 133 +--
 .../localmeta/table/EDW.TEST_CAL_DT.json| 814 +--
 .../table/EDW.TEST_SELLER_TYPE_DIM.json |  86 +-
 .../table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json   |  43 +
 .../localmeta/table/EDW.TEST_SITES.json |  94 +--
 .../localmeta/table/EDW.V_TEST_CAL_DT.json  | 409 --
 .../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json  | 814 +--
 .../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json  | 298 +++
 .../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json |  46 +-
 .../localmeta/table/SNOWTEST.KYLIN_SALES.json   | 112 +--
 .../kylin/provision/BuildCubeWithEngine.java|  83 +-
 .../java/org/apache/kylin/query/H2Database.java |  14 +-
 .../apache/kylin/query/ITKylinQueryTest.java|   2 +-
 .../org/apache/kylin/query/KylinTestBase.java   |   1 -
 29 files changed, 1498 insertions(+), 2480 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 8fc583d..c8b0d43 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -122,13 +122,13 @@ public class DeployUtil {
 // 

 
 static final String TABLE_CAL_DT = "edw.test_cal_dt";
-static final String VIEW_CAL_DT = "edw.v_test_cal_dt";
 static final String TABLE_CATEGORY_GROUPINGS = 
"default.test_category_groupings";
 static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
-static final String TABLE_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
+static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
+static final String TABLE_SELLER_TYPE_DIM_TABLE = 
"edw.test_seller_type_dim_table";
 static final String TABLE_SITES = "edw.test_sites";
 
-static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, 
TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM, TABLE_SITES 
};
+static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, 
TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, 
TABLE_SITES };
 
 public static void prepareTestDataForNormalCubes(String cubeName) throws 
Exception {
 
@@ -230,7 +230,7 @@ public class DeployUtil {
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase(;
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase(;
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase(;
-
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM.toUpperCase(;
+
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase(;
 
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase(;
 
 // load data to hive tables
@@ -238,11 +238,11 @@ public class DeployUtil {
 hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT,

[2/2] kylin git commit: KYLIN-2287 minor, UI enhancement

[1/2] kylin git commit: KYLIN-2287 Speed up model and cube list load in Web

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[1/4] kylin git commit: minor, fix sample model [Forced Update!]

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[1/3] kylin git commit: minor, fix sample model [Forced Update!]

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

kylin git commit: minor, fix sample model

kylin git commit: try pass UT

kylin git commit: minor, fix the typo

[02/15] kylin git commit: KYLIN-2283 A new data gen tool for CI

[05/15] kylin git commit: KYLIN-2288 recogonize measure empty string as zero

[07/15] kylin git commit: KYLIN-2297, Manually edit cube segment start/end time will throw error in UI

[08/15] kylin git commit: KYLIN-2298, timer component get wrong seconds

[14/15] kylin git commit: KYLIN-2269 Reduce MR memory usage for global dict

[13/15] kylin git commit: KYLIN 1875 minor, update model ang cube designer

[10/15] kylin git commit: KYLIN-2292 workaround for CALCITE-1540

[11/15] kylin git commit: KYLIN-2295 Refactor CI, blend view cubes into the rest

[03/15] kylin git commit: KYLIN-2283 A new data gen tool for CI

[01/15] kylin git commit: KYLIN-2283 A new data gen tool for CI [Forced Update!]

[04/15] kylin git commit: KYLIN-2283 A new data gen tool for CI

[15/15] kylin git commit: KYLIN-2293 remove the hack set methods in KylinConfig

[06/15] kylin git commit: minor, add equals and hashcode to RowKeyColDesc

[09/15] kylin git commit: KYLIN-2290 minor improvements on limit

kylin git commit: pass UT

[kylin] Git Push Summary

kylin git commit: add CI metadata, pending cube desc

kylin git commit: KYLIN-2269 Reduce MR memory usage for global dict

kylin git commit: KYLIN 1875 minor,update model ang cube designer

kylin git commit: KYLIN-1770, remove DateRangeRules

[1/2] kylin git commit: KYLIN-2295 Refactor CI, blend view cubes into the rest

[2/2] kylin git commit: KYLIN-2295 Refactor CI, blend view cubes into the rest

34 matches

Site Navigation

Mail list logo

Footer information