date:20161222

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-22 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/afec45c5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/afec45c5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

[1/4] kylin git commit: minor code refactors [Forced Update!]

2016-12-22 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 6f1a7b730 -> 13f41daaf (forced update)


minor code refactors


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c77a17f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c77a17f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c77a17f

Branch: refs/heads/yang22-hbase102
Commit: 7c77a17f56404d8c263254b8567c1ec105a85ff6
Parents: 85313ee
Author: Hongbin Ma 
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 23 11:32:13 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 ++-
 .../org/apache/kylin/common/util/BasicTest.java |  1 +
 .../gtrecord/DummyPartitionStreamer.java| 40 
 .../storage/gtrecord/IPartitionStreamer.java| 26 +
 .../gtrecord/StorageResponseGTScatter.java  |  9 +++--
 .../apache/kylin/rest/service/QueryService.java |  6 ++-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  9 +++--
 7 files changed, 87 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a1c9050..ceda95c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -782,7 +782,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
 }
 
-@Deprecated//Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
+@Deprecated //Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }
@@ -791,6 +791,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan-threshold", 
"1000"));
 }
 
+public int getLargeQueryThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.large-query-threshold", 
String.valueOf((int) (getScanThreshold() * 0.1;
+}
+
 public int getDerivedInThreshold() {
 return 
Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold",
 "20"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 000..4caaed0
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT

[1/4] kylin git commit: minor code refactors [Forced Update!]

2016-12-22 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 74658e925 -> 42002f605 (forced update)


minor code refactors


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c77a17f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c77a17f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c77a17f

Branch: refs/heads/yang22-cdh5.7
Commit: 7c77a17f56404d8c263254b8567c1ec105a85ff6
Parents: 85313ee
Author: Hongbin Ma 
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 23 11:32:13 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 ++-
 .../org/apache/kylin/common/util/BasicTest.java |  1 +
 .../gtrecord/DummyPartitionStreamer.java| 40 
 .../storage/gtrecord/IPartitionStreamer.java| 26 +
 .../gtrecord/StorageResponseGTScatter.java  |  9 +++--
 .../apache/kylin/rest/service/QueryService.java |  6 ++-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  9 +++--
 7 files changed, 87 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a1c9050..ceda95c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -782,7 +782,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
 }
 
-@Deprecated//Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
+@Deprecated //Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }
@@ -791,6 +791,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan-threshold", 
"1000"));
 }
 
+public int getLargeQueryThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.large-query-threshold", 
String.valueOf((int) (getScanThreshold() * 0.1;
+}
+
 public int getDerivedInThreshold() {
 return 
Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold",
 "20"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 000..4caaed0
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-22 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/afec45c5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/afec45c5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/afec45c5

Branch: refs/heads/yang22-cdh5.7
Commit: afec45c55b481a5be82e5ae3c6a9ae6b82248cf2
Parents: 7c77a17
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 15:53:11 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/afec45c5/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-12-22 Thread billyliu

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/42002f60
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/42002f60
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/42002f60

Branch: refs/heads/yang22-cdh5.7
Commit: 42002f60528d91d62d99d693e5adab378cd2a203
Parents: afec45c
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Fri Dec 23 15:54:56 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/42002f60/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/42002f60/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/42002f60/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-22 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/afec45c5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/afec45c5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-22 Thread billyliu

http://git-wip-us.apache.org/repos/asf/kylin/blob/afec45c5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/afec45c5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

[1/3] kylin git commit: minor code refactors [Forced Update!]

2016-12-22 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x c677e5409 -> afec45c55 (forced update)


minor code refactors


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c77a17f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c77a17f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c77a17f

Branch: refs/heads/yang22-hbase1.x
Commit: 7c77a17f56404d8c263254b8567c1ec105a85ff6
Parents: 85313ee
Author: Hongbin Ma 
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 23 11:32:13 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 ++-
 .../org/apache/kylin/common/util/BasicTest.java |  1 +
 .../gtrecord/DummyPartitionStreamer.java| 40 
 .../storage/gtrecord/IPartitionStreamer.java| 26 +
 .../gtrecord/StorageResponseGTScatter.java  |  9 +++--
 .../apache/kylin/rest/service/QueryService.java |  6 ++-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  9 +++--
 7 files changed, 87 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a1c9050..ceda95c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -782,7 +782,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
 }
 
-@Deprecated//Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
+@Deprecated //Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }
@@ -791,6 +791,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan-threshold", 
"1000"));
 }
 
+public int getLargeQueryThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.large-query-threshold", 
String.valueOf((int) (getScanThreshold() * 0.1;
+}
+
 public int getDerivedInThreshold() {
 return 
Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold",
 "20"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 000..4caaed0
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-22 Thread billyliu

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/afec45c5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/afec45c5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/afec45c5

Branch: refs/heads/yang22-hbase1.x
Commit: afec45c55b481a5be82e5ae3c6a9ae6b82248cf2
Parents: 7c77a17
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Fri Dec 23 15:53:11 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/afec45c5/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

kylin git commit: add more log

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/xiefan22 74658e925 -> fde2e0a5c


add more log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fde2e0a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fde2e0a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fde2e0a5

Branch: refs/heads/xiefan22
Commit: fde2e0a5c2c2fa8100fe96e3e3c4f3315dfab58b
Parents: 74658e9
Author: Hongbin Ma 
Authored: Fri Dec 23 13:56:01 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 23 13:56:01 2016 +0800

--
 .../apache/kylin/storage/gtrecord/StorageResponseGTScatter.java| 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fde2e0a5/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index fe1afd3..0e7809a 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -74,8 +74,10 @@ public class StorageResponseGTScatter implements IGTScanner {
 public Iterator iterator() {
 Iterator shardSubsets = 
Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
 if (storagePushDownLimit != Integer.MAX_VALUE) {
+logger.info("Using SortedIteratorMergerWithLimit to merge 
partitions");
 return new SortedIteratorMergerWithLimit(shardSubsets, 
storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
 } else {
+logger.info("Using Iterators.concat to merge partitions");
 return Iterators.concat(shardSubsets);
 }
 }

[kylin] Git Push Summary

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/xiefan22 [created] 74658e925

[kylin] Git Push Summary

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/xiefan22 [deleted] 85313ee63

[kylin] Git Push Summary

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/xiefan22 [created] 85313ee63

kylin git commit: minor code refactors

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang22 85313ee63 -> 7c77a17f5


minor code refactors


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c77a17f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c77a17f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c77a17f

Branch: refs/heads/yang22
Commit: 7c77a17f56404d8c263254b8567c1ec105a85ff6
Parents: 85313ee
Author: Hongbin Ma 
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 23 11:32:13 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 ++-
 .../org/apache/kylin/common/util/BasicTest.java |  1 +
 .../gtrecord/DummyPartitionStreamer.java| 40 
 .../storage/gtrecord/IPartitionStreamer.java| 26 +
 .../gtrecord/StorageResponseGTScatter.java  |  9 +++--
 .../apache/kylin/rest/service/QueryService.java |  6 ++-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  9 +++--
 7 files changed, 87 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a1c9050..ceda95c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -782,7 +782,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
 }
 
-@Deprecated//Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
+@Deprecated //Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }
@@ -791,6 +791,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan-threshold", 
"1000"));
 }
 
+public int getLargeQueryThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.large-query-threshold", 
String.valueOf((int) (getScanThreshold() * 0.1;
+}
+
 public int getDerivedInThreshold() {
 return 
Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold",
 "20"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c77a17f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 000..4caaed0
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,

kylin git commit: KYLIN-2310 ignore ITDoggedCubeBuilderStressTest to save CI time

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 4d1676a66 -> 120df2d55


KYLIN-2310 ignore ITDoggedCubeBuilderStressTest to save CI time


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/120df2d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/120df2d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/120df2d5

Branch: refs/heads/master
Commit: 120df2d55f2e5aec742be646046a6654bcf5492b
Parents: 4d1676a
Author: Li Yang 
Authored: Fri Dec 23 08:15:37 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 23 08:17:49 2016 +0800

--
 .../kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java  | 2 ++
 .../org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/120df2d5/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
index 4d23979..0c32d18 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
@@ -37,6 +37,7 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -44,6 +45,7 @@ import org.slf4j.LoggerFactory;
 /**
  * very time consuming
  */
+@Ignore("to save CI time")
 public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
 
 @SuppressWarnings("unused")

http://git-wip-us.apache.org/repos/asf/kylin/blob/120df2d5/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
index 806bfab..1db7a24 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
@@ -57,7 +57,7 @@ public class ITDoggedCubeBuilderTest extends 
LocalFileMetadataTestCase {
 @SuppressWarnings("unused")
 private static final Logger logger = 
LoggerFactory.getLogger(ITDoggedCubeBuilderTest.class);
 
-private static final int INPUT_ROWS = 4000;
+private static final int INPUT_ROWS = 1;
 private static final int SPLIT_ROWS = 2000;
 private static final int THREADS = 4;

[2/2] kylin git commit: temp2

2016-12-22 Thread liyang

temp2


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f1a7a86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f1a7a86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f1a7a86

Branch: refs/heads/mhb22
Commit: 1f1a7a8698024621e5b7f84cfe37eb23e691c9dd
Parents: 8f6fbd4
Author: Hongbin Ma 
Authored: Thu Dec 22 20:30:54 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 22 20:33:52 2016 +0800

--
 .../org/apache/kylin/common/util/BasicTest.java |  1 +
 .../gtrecord/DummyPartitionStreamer.java| 40 
 .../storage/gtrecord/IPartitionStreamer.java| 26 +
 .../gtrecord/StorageResponseGTScatter.java  |  8 ++--
 .../apache/kylin/rest/service/QueryService.java | 26 +++--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  9 +++--
 6 files changed, 99 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1f1a7a86/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import com.google.common.collect.Iterators;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1f1a7a86/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 000..4caaed0
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class DummyPartitionStreamer implements IPartitionStreamer {
+private Iterator iterator;
+
+public DummyPartitionStreamer(Iterator iterator) {
+this.iterator = iterator;
+}
+
+@Override
+public void close() throws IOException {
+//do nothing
+}
+
+@Override
+public Iterator asByteArrayIterator() {
+return this.iterator;
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/1f1a7a86/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
new file mode 100644
index 000..42f1151
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed

[1/2] kylin git commit: temp

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/mhb22 [created] 1f1a7a869


temp


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8f6fbd47
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8f6fbd47
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8f6fbd47

Branch: refs/heads/mhb22
Commit: 8f6fbd478430170090203c79ee4d256c838b31b8
Parents: 85313ee
Author: Hongbin Ma 
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 22 20:33:35 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  6 +++-
 .../gtrecord/StorageResponseGTScatter.java  |  3 +-
 .../apache/kylin/rest/service/QueryService.java | 30 +---
 3 files changed, 14 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a1c9050..ceda95c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -782,7 +782,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
 }
 
-@Deprecated//Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
+@Deprecated //Limit is good even it's large. This config is meaning less 
since we already have scan threshold 
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }
@@ -791,6 +791,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.scan-threshold", 
"1000"));
 }
 
+public int getLargeQueryThreshold() {
+return 
Integer.parseInt(getOptional("kylin.query.large-query-threshold", 
String.valueOf((int) (getScanThreshold() * 0.1;
+}
+
 public int getDerivedInThreshold() {
 return 
Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold",
 "20"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index fe1afd3..88116b5 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -67,7 +67,8 @@ public class StorageResponseGTScatter implements IGTScanner {
 
 @Override
 public void close() throws IOException {
-//do nothing
+//If upper consumer failed while consuming the GTRecords, the consumer 
should call IGTScanner's close method to ensure releasing resource
+Iterators.
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8810c85..82bc407 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -19,27 +19,8 @@
 package org.apache.kylin.rest.service;
 
 import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import java.sql.*;
+import java.util.*;
 
 import javax.annotation.PostConstruct;
 import javax.sql.DataSource;
@@

kylin git commit: KYLIN-2312 display server config/environment by order in system tab

2016-12-22 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/master 0d46e4a27 -> 4d1676a66


KYLIN-2312 display server config/environment by order in system tab


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d1676a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d1676a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d1676a6

Branch: refs/heads/master
Commit: 4d1676a66889283647b7c63022505234fa9d328c
Parents: 0d46e4a
Author: Billy Liu 
Authored: Thu Dec 22 18:55:51 2016 +0800
Committer: Billy Liu 
Committed: Thu Dec 22 18:55:51 2016 +0800

--
 .../common/BackwardCompatibilityConfig.java |   9 +
 .../org/apache/kylin/common/KylinConfig.java|  21 +-
 .../kylin/common/util/OrderedProperties.java| 388 +++
 .../kylin/job/constant/ExecutableConstants.java |   4 -
 .../apache/kylin/rest/service/AdminService.java |  16 +-
 5 files changed, 417 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1676a6/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
 
b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
index 21df932..426ebb9 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java
@@ -29,6 +29,7 @@ import java.util.Properties;
 import java.util.Stack;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.util.OrderedProperties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -121,6 +122,14 @@ public class BackwardCompatibilityConfig {
 return result;
 }
 
+public OrderedProperties check(OrderedProperties props){
+OrderedProperties result = new OrderedProperties();
+for (Entry kv : props.entrySet()) {
+result.setProperty(check(kv.getKey()), kv.getValue());
+}
+return result;
+}
+
 // 

 
 public static void main(String[] args) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1676a6/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index acd4398..f169142 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -25,7 +25,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintWriter;
 import java.io.StringReader;
-import java.io.StringWriter;
 import java.nio.charset.Charset;
 import java.util.Enumeration;
 import java.util.Map;
@@ -35,6 +34,7 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.restclient.RestClient;
 import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.OrderedProperties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -282,18 +282,15 @@ public class KylinConfig extends KylinConfigBase {
 }
 
 public String getConfigAsString() throws IOException {
-final StringWriter stringWriter = new StringWriter();
-list(new PrintWriter(stringWriter));
-return stringWriter.toString();
-}
-
-private void list(PrintWriter out) {
-Properties props = getAllProperties();
-for (Enumeration e = props.keys(); e.hasMoreElements();) {
-String key = (String) e.nextElement();
-String val = (String) props.get(key);
-out.println(key + "=" + val);
+File propertiesFile = getKylinPropertiesFile();
+OrderedProperties orderedProperties = new OrderedProperties();
+orderedProperties.load(new FileInputStream(propertiesFile));
+orderedProperties = BCC.check(orderedProperties);
+final StringBuilder sb = new StringBuilder();
+for (Map.Entry entry : orderedProperties.entrySet()) {
+sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
 }
+return sb.toString();
 }
 
 public KylinConfig base() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1676a6/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java

[kylin] Git Push Summary

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2310 [deleted] 320baf929

kylin git commit: KYLIN-2310 Alter CI cube, add test for date/time encoding and extended column

2016-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 440394de1 -> 0d46e4a27


KYLIN-2310 Alter CI cube, add test for date/time encoding and extended column


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0d46e4a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0d46e4a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0d46e4a2

Branch: refs/heads/master
Commit: 0d46e4a270749475a1154db94391970748fa3a8e
Parents: 440394d
Author: Li Yang 
Authored: Thu Dec 22 13:37:57 2016 +0800
Committer: Li Yang 
Committed: Thu Dec 22 16:07:58 2016 +0800

--
 .../kylin/source/datagen/DataGenTest.java   |  2 +-
 .../localmeta/cube_desc/ci_inner_join_cube.json | 31 +++-
 .../localmeta/cube_desc/ci_left_join_cube.json  | 31 +++-
 .../model_desc/ci_inner_join_model.json |  8 +++--
 .../model_desc/ci_left_join_model.json  |  6 ++--
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |  5 
 .../localmeta/table/DEFAULT.TEST_ORDER.json | 15 ++
 .../java/org/apache/kylin/query/H2Database.java |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java| 14 +++--
 .../source/hive/ITHiveTableReaderTest.java  |  2 +-
 .../resources/query/sql_all_types/query00.sql   | 19 
 .../resources/query/sql_all_types/query01.sql   | 19 
 .../resources/query/sql_all_types/query02.sql   | 19 
 .../resources/query/sql_all_types/query03.sql   | 19 
 .../resources/query/sql_all_types/query04.sql   | 19 
 .../resources/query/sql_all_types/query05.sql   | 19 
 .../resources/query/sql_all_types/query06.sql   | 19 
 .../resources/query/sql_all_types/query07.sql   | 19 
 .../resources/query/sql_all_types/query08.sql   | 19 
 .../resources/query/sql_all_types/query09.sql   | 19 
 .../resources/query/sql_datetime/query00.sql| 14 +
 .../query/sql_extended_column/query00.sql   | 12 
 .../resources/query/sql_fast_common/query00.sql | 23 ---
 .../src/test/resources/query/sql_ii/query01.sql | 19 
 .../src/test/resources/query/sql_ii/query02.sql | 19 
 .../src/test/resources/query/sql_ii/query03.sql | 19 
 .../src/test/resources/query/sql_ii/query04.sql | 19 
 .../src/test/resources/query/sql_ii/query05.sql | 19 
 .../resources/query/sql_ii/query06.sql.disabled | 19 
 .../resources/query/sql_ii/query07.sql.disabled | 19 
 .../src/test/resources/query/sql_ii/query08.sql | 19 
 .../src/test/resources/query/sql_ii/query09.sql | 19 
 32 files changed, 114 insertions(+), 412 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0d46e4a2/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java 
b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
index 82455ab..5e3ca9f 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
@@ -42,7 +42,7 @@ public class DataGenTest extends LocalFileMetadataTestCase {
 
 @Test
 public void testCIConfigured() throws IOException {
-DataModelDesc model = getModel("test_kylin_inner_join_model_desc");
+DataModelDesc model = getModel("ci_inner_join_model");
 ModelDataGenerator gen = new ModelDataGenerator(model, 100);
 gen.outprint = true;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d46e4a2/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json 
b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 48a051d..287ead6 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -4,15 +4,23 @@
   "model_name" : "ci_inner_join_model",
   "description" : null,
   "dimensions" : [ {
-"name" : "ORDER_ID",
-"table" : "DEFAULT.TEST_KYLIN_FACT",
-"column" : "ORDER_ID"
-  }, {
 "name" : "CAL_DT",
 "table" : "EDW.TEST_CAL_DT",
 "column" : "{FK}",
 "derived" : [ "WEEK_BEG_DT" ]
   }, {
+"name" : "ORDER_ID",
+"table" : "DEFAULT.TEST_KYLIN_FACT",
+"column" : "ORDER_ID"
+  }, {
+"name" : "TEST_DATE_ENC",
+"table" : "DEFAULT.TEST_ORDER",
+"column" :

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[1/4] kylin git commit: minor code refactors [Forced Update!]

[1/4] kylin git commit: minor code refactors [Forced Update!]

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[1/3] kylin git commit: minor code refactors [Forced Update!]

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

kylin git commit: add more log

[kylin] Git Push Summary

[kylin] Git Push Summary

[kylin] Git Push Summary

kylin git commit: minor code refactors

kylin git commit: KYLIN-2310 ignore ITDoggedCubeBuilderStressTest to save CI time

[2/2] kylin git commit: temp2

[1/2] kylin git commit: temp

kylin git commit: KYLIN-2312 display server config/environment by order in system tab

[kylin] Git Push Summary

kylin git commit: KYLIN-2310 Alter CI cube, add test for date/time encoding and extended column

20 matches

Site Navigation

Mail list logo

Footer information