date:20161206

kylin git commit: minor, update calculateCuboid API to POST

2016-12-06 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22 8a42973d3 -> 43805bcd2


minor, update calculateCuboid API to POST


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/43805bcd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/43805bcd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/43805bcd

Branch: refs/heads/yang22
Commit: 43805bcd2c3bb49d5b1c433f6205927d36a7eaad
Parents: 8a42973
Author: Billy Liu 
Authored: Wed Dec 7 15:49:17 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 15:49:17 2016 +0800

--
 .../main/java/org/apache/kylin/rest/controller/CubeController.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/43805bcd/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8dce112..4c07f7c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -652,7 +652,7 @@ public class CubeController extends BasicController {
  * @param aggregationGroupStr
  * @return number of cuboid, -1 if failed
  */
-@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.GET)
+@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.POST)
 @ResponseBody
 public long calculateCuboidCombination(@RequestBody String 
aggregationGroupStr) {
 AggregationGroup aggregationGroup = 
deserializeAggregationGroup(aggregationGroupStr);

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83862138
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83862138
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83862138

Branch: refs/heads/yang22-cdh5.7
Commit: 838621385fb493d5b2d2703afaae3d091e07ae20
Parents: 8a42973
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 15:34:08 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83862138/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-12-06 Thread lidong

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7edf9d07
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7edf9d07
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7edf9d07

Branch: refs/heads/yang22-cdh5.7
Commit: 7edf9d07f8de2cb0745330c58c64c42a5aba4f88
Parents: 8386213
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Wed Dec 7 15:35:43 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7edf9d07/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/7edf9d07/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7edf9d07/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2016-12-06 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 fda38a9b5 -> 7edf9d07f (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/83862138/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/83862138/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath =

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2016-12-06 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 9089db49f -> 838621385 (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/83862138/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/83862138/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath =

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83862138
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83862138
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83862138

Branch: refs/heads/yang22-hbase1.x
Commit: 838621385fb493d5b2d2703afaae3d091e07ae20
Parents: 8a42973
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 15:34:08 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83862138/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/4] kylin git commit: minor, Get cuboid combination number from cube API [Forced Update!]

2016-12-06 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 a58cefaae -> fda38a9b5 (forced update)


minor, Get cuboid combination number from cube API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8a42973d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8a42973d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8a42973d

Branch: refs/heads/yang22-cdh5.7
Commit: 8a42973d329ccf61f9b7e4b56246dd6ce2501db8
Parents: 3a120f3
Author: Billy Liu 
Authored: Wed Dec 7 14:02:43 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 14:02:43 2016 +0800

--
 .../kylin/rest/controller/CubeController.java   | 33 
 1 file changed, 33 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8a42973d/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 3846d28..8dce112 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -645,6 +646,22 @@ public class CubeController extends BasicController {
 return response;
 }
 
+/**
+ * Calculate Cuboid Combination based on the AggreationGroup definition.
+ *
+ * @param aggregationGroupStr
+ * @return number of cuboid, -1 if failed
+ */
+@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.GET)
+@ResponseBody
+public long calculateCuboidCombination(@RequestBody String 
aggregationGroupStr) {
+AggregationGroup aggregationGroup = 
deserializeAggregationGroup(aggregationGroupStr);
+if (aggregationGroup != null) {
+return aggregationGroup.calculateCuboidCombination();
+} else
+return -1;
+}
+
 private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
 CubeDesc desc = null;
 try {
@@ -663,6 +680,22 @@ public class CubeController extends BasicController {
 return desc;
 }
 
+private AggregationGroup deserializeAggregationGroup(String 
aggregationGroupStr) {
+AggregationGroup aggreationGroup = null;
+try {
+logger.debug("Parsing AggregationGroup " + aggregationGroupStr);
+aggreationGroup = JsonUtil.readValue(aggregationGroupStr, 
AggregationGroup.class);
+} catch (JsonParseException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (JsonMappingException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (IOException e) {
+logger.error("Failed to deal with the request.", e);
+throw new InternalErrorException("Failed to deal with the 
request:" + e.getMessage(), e);
+}
+return aggreationGroup;
+}
+
 private void updateRequest(CubeRequest request, boolean success, String 
message) {
 request.setCubeDescData("");
 request.setSuccessful(success);

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9089db49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9089db49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9089db49

Branch: refs/heads/yang22-cdh5.7
Commit: 9089db49f8b21dd5e894f7057289ffd3954eed99
Parents: 8a42973
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 14:36:22 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9089db49/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-12-06 Thread lidong

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fda38a9b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fda38a9b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fda38a9b

Branch: refs/heads/yang22-cdh5.7
Commit: fda38a9b5914594b198cf7123116a5231d9f0468
Parents: 9089db4
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Wed Dec 7 14:38:02 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fda38a9b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/fda38a9b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fda38a9b/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/9089db49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/9089db49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9089db49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9089db49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9089db49

Branch: refs/heads/yang22-hbase1.x
Commit: 9089db49f8b21dd5e894f7057289ffd3954eed99
Parents: 8a42973
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 14:36:22 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9089db49/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

2016-12-06 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 59705faa7 -> 9089db49f (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/9089db49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/9089db49/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath =

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/59705faa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/59705faa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59705faa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59705faa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59705faa

Branch: refs/heads/yang22-hbase1.x
Commit: 59705faa7991b8b2a65fcd6a433457bf3dc0fcfb
Parents: 8a42973
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 14:11:22 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59705faa/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[1/3] kylin git commit: minor, Get cuboid combination number from cube API [Forced Update!]

2016-12-06 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x dd5e28cd8 -> 59705faa7 (forced update)


minor, Get cuboid combination number from cube API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8a42973d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8a42973d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8a42973d

Branch: refs/heads/yang22-hbase1.x
Commit: 8a42973d329ccf61f9b7e4b56246dd6ce2501db8
Parents: 3a120f3
Author: Billy Liu 
Authored: Wed Dec 7 14:02:43 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 14:02:43 2016 +0800

--
 .../kylin/rest/controller/CubeController.java   | 33 
 1 file changed, 33 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8a42973d/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 3846d28..8dce112 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -645,6 +646,22 @@ public class CubeController extends BasicController {
 return response;
 }
 
+/**
+ * Calculate Cuboid Combination based on the AggreationGroup definition.
+ *
+ * @param aggregationGroupStr
+ * @return number of cuboid, -1 if failed
+ */
+@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.GET)
+@ResponseBody
+public long calculateCuboidCombination(@RequestBody String 
aggregationGroupStr) {
+AggregationGroup aggregationGroup = 
deserializeAggregationGroup(aggregationGroupStr);
+if (aggregationGroup != null) {
+return aggregationGroup.calculateCuboidCombination();
+} else
+return -1;
+}
+
 private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
 CubeDesc desc = null;
 try {
@@ -663,6 +680,22 @@ public class CubeController extends BasicController {
 return desc;
 }
 
+private AggregationGroup deserializeAggregationGroup(String 
aggregationGroupStr) {
+AggregationGroup aggreationGroup = null;
+try {
+logger.debug("Parsing AggregationGroup " + aggregationGroupStr);
+aggreationGroup = JsonUtil.readValue(aggregationGroupStr, 
AggregationGroup.class);
+} catch (JsonParseException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (JsonMappingException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (IOException e) {
+logger.error("Failed to deal with the request.", e);
+throw new InternalErrorException("Failed to deal with the 
request:" + e.getMessage(), e);
+}
+return aggreationGroup;
+}
+
 private void updateRequest(CubeRequest request, boolean success, String 
message) {
 request.setCubeDescData("");
 request.setSuccessful(success);

kylin git commit: minor, Get cuboid combination number from cube API

2016-12-06 Thread billyliu

Repository: kylin
Updated Branches:
  refs/heads/yang22 3a120f3e6 -> 8a42973d3


minor, Get cuboid combination number from cube API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8a42973d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8a42973d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8a42973d

Branch: refs/heads/yang22
Commit: 8a42973d329ccf61f9b7e4b56246dd6ce2501db8
Parents: 3a120f3
Author: Billy Liu 
Authored: Wed Dec 7 14:02:43 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 14:02:43 2016 +0800

--
 .../kylin/rest/controller/CubeController.java   | 33 
 1 file changed, 33 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8a42973d/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 3846d28..8dce112 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -645,6 +646,22 @@ public class CubeController extends BasicController {
 return response;
 }
 
+/**
+ * Calculate Cuboid Combination based on the AggreationGroup definition.
+ *
+ * @param aggregationGroupStr
+ * @return number of cuboid, -1 if failed
+ */
+@RequestMapping(value = "aggregationgroups/cuboid", method = 
RequestMethod.GET)
+@ResponseBody
+public long calculateCuboidCombination(@RequestBody String 
aggregationGroupStr) {
+AggregationGroup aggregationGroup = 
deserializeAggregationGroup(aggregationGroupStr);
+if (aggregationGroup != null) {
+return aggregationGroup.calculateCuboidCombination();
+} else
+return -1;
+}
+
 private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
 CubeDesc desc = null;
 try {
@@ -663,6 +680,22 @@ public class CubeController extends BasicController {
 return desc;
 }
 
+private AggregationGroup deserializeAggregationGroup(String 
aggregationGroupStr) {
+AggregationGroup aggreationGroup = null;
+try {
+logger.debug("Parsing AggregationGroup " + aggregationGroupStr);
+aggreationGroup = JsonUtil.readValue(aggregationGroupStr, 
AggregationGroup.class);
+} catch (JsonParseException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (JsonMappingException e) {
+logger.error("The AggregationGroup definition is not valid.", e);
+} catch (IOException e) {
+logger.error("Failed to deal with the request.", e);
+throw new InternalErrorException("Failed to deal with the 
request:" + e.getMessage(), e);
+}
+return aggreationGroup;
+}
+
 private void updateRequest(CubeRequest request, boolean success, String 
message) {
 request.setCubeDescData("");
 request.setSuccessful(success);

kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-12-06 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 [created] a58cefaae


KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a58cefaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a58cefaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a58cefaa

Branch: refs/heads/yang22-cdh5.7
Commit: a58cefaae63c33d4e3671b904a42b000c5a05083
Parents: dd5e28c
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Wed Dec 7 12:34:20 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a58cefaa/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/a58cefaa/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a58cefaa/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x [created] dd5e28cd8


http://git-wip-us.apache.org/repos/asf/kylin/blob/dd5e28cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/dd5e28cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath =

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dd5e28cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dd5e28cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dd5e28cd

Branch: refs/heads/yang22-hbase1.x
Commit: dd5e28cd844344b7f3aba0ac0c120a096f488f19
Parents: 3a120f3
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 12:33:49 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dd5e28cd/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[34/50] [abbrv] kylin git commit: KYLIN-2236 let query cache honor backdoortoggles in query request

2016-12-06 Thread lidong

KYLIN-2236 let query cache honor backdoortoggles in query request


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8f3239bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8f3239bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8f3239bf

Branch: refs/heads/master-cdh5.7
Commit: 8f3239bf9ea4f1abd8d2c814967d425d4da68a45
Parents: 9556152
Author: Hongbin Ma 
Authored: Mon Dec 5 13:43:03 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 5 13:43:03 2016 +0800

--
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 30 ++
 .../apache/kylin/rest/request/SQLRequest.java   | 63 +++-
 .../apache/kylin/rest/service/QueryService.java |  1 +
 3 files changed, 53 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3239bf/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index 4c847bf..2f8991b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -261,6 +261,36 @@ public class ITJDBCDriverTest extends 
HBaseMetadataTestCase {
 
 }
 
+
+@Test
+public void testResultSetWithMaxRows() throws Exception {
+String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as 
TRANS_CNT from test_kylin_fact \n" + " group by LSTG_FORMAT_NAME ";
+
+Connection conn = getConnection();
+Statement statement = conn.createStatement();
+statement.setMaxRows(2);
+
+statement.execute(sql);
+
+ResultSet rs = statement.getResultSet();
+
+int count = 0;
+while (rs.next()) {
+count++;
+String lstg = rs.getString(1);
+double gmv = rs.getDouble(2);
+int trans_count = rs.getInt(3);
+
+System.out.println("Get a line: LSTG_FORMAT_NAME=" + lstg + ", 
GMV=" + gmv + ", TRANS_CNT=" + trans_count);
+}
+
+Assert.assertTrue(count == 2);
+statement.close();
+rs.close();
+conn.close();
+
+}
+
 private static class SystemPropertiesOverride {
 HashMap backup = new HashMap();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3239bf/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java 
b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index 96f5faa..bd8b7e2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -83,50 +83,31 @@ public class SQLRequest implements Serializable {
 this.acceptPartial = acceptPartial;
 }
 
+
 @Override
-public int hashCode() {
-final int prime = 31;
-int result = 1;
-result = prime * result + (acceptPartial ? 1231 : 1237);
-result = prime * result + ((offset == null) ? 0 : offset.hashCode());
-result = prime * result + ((limit == null) ? 0 : limit.hashCode());
-result = prime * result + ((project == null) ? 0 : project.hashCode());
-result = prime * result + ((sql == null) ? 0 : sql.hashCode());
-return result;
+public boolean equals(Object o) {
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
+
+SQLRequest that = (SQLRequest) o;
+
+if (acceptPartial != that.acceptPartial) return false;
+if (sql != null ? !sql.equals(that.sql) : that.sql != null) return 
false;
+if (project != null ? !project.equals(that.project) : that.project != 
null) return false;
+if (offset != null ? !offset.equals(that.offset) : that.offset != 
null) return false;
+if (limit != null ? !limit.equals(that.limit) : that.limit != null) 
return false;
+return backdoorToggles != null ? 
backdoorToggles.equals(that.backdoorToggles) : that.backdoorToggles == null;
+
 }
 
 @Override
-public boolean equals(Object obj) {
-if (this == obj)
-return true;
-if (obj == null)
-return false;
-if (getClass() != obj.getClass())
-return false;
-SQLRequest other = (SQLRequest) obj;
-if (acceptPartial != other.acceptPartial)
-return false;
-if (offset == null) {
-if (other.offset

[11/50] [abbrv] kylin git commit: filter null value in cubedesc and model desc

2016-12-06 Thread lidong

filter null value in cubedesc and model desc

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/875a5931
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/875a5931
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/875a5931

Branch: refs/heads/master-cdh5.7
Commit: 875a5931ad8cad79cdec1e80ab9388ded9446091
Parents: 3186d17
Author: luguosheng <550175...@qq.com>
Authored: Wed Nov 30 10:51:51 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 30 13:48:29 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js  |  4 ++--
 webapp/app/js/controllers/modelEdit.js | 13 ++---
 webapp/app/js/utils/utils.js   | 24 
 3 files changed, 36 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 888662d..0704a62 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -413,7 +413,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 
 if ($scope.isEdit) {
   CubeService.update({}, {
-cubeDescData: $scope.state.cubeSchema,
+cubeDescData: VdmUtil.filterNullValInObj($scope.state.cubeSchema),
 cubeName: $routeParams.cubeName,
 project: $scope.state.project
   }, function (request) {
@@ -453,7 +453,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   });
 } else {
   CubeService.save({}, {
-cubeDescData: $scope.state.cubeSchema,
+cubeDescData: VdmUtil.filterNullValInObj($scope.state.cubeSchema),
 project: $scope.state.project
   }, function (request) {
 if (request.successful) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/controllers/modelEdit.js
--
diff --git a/webapp/app/js/controllers/modelEdit.js 
b/webapp/app/js/controllers/modelEdit.js
index 0b3db66..2f1b35e 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -19,7 +19,7 @@
 'use strict';
 
 
-KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, 
$location, $templateCache, $interpolate, MessageService, TableService, 
CubeDescService, ModelService, loadingRequest, 
SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager)
 {
+KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, 
$location, $templateCache, $interpolate, MessageService, TableService, 
CubeDescService, ModelService, loadingRequest, 
SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager,VdmUtil)
 {
 //add or edit ?
 var absUrl = $location.absUrl();
 $scope.modelMode = 
absUrl.indexOf("/models/add")!=-1?'addNewModel':absUrl.indexOf("/models/edit")!=-1?'editExistModel':'default';
@@ -188,7 +188,11 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, 
$routeParams, $locati
 loadingRequest.show();
 
 if ($scope.isEdit) {
-ModelService.update({}, 
{modelDescData:$scope.state.modelSchema, modelName: $routeParams.modelName, 
project: $scope.state.project}, function (request) {
+ModelService.update({}, {
+  
modelDescData:VdmUtil.filterNullValInObj($scope.state.modelSchema),
+  modelName: $routeParams.modelName,
+  project: $scope.state.project
+}, function (request) {
 if (request.successful) {
 $scope.state.modelSchema = request.modelSchema;
 SweetAlert.swal('', 'Updated the model 
successfully.', 'success');
@@ -216,7 +220,10 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, 
$routeParams, $locati
 loadingRequest.hide();
 });
 } else {
-ModelService.save({}, 
{modelDescData:$scope.state.modelSchema, project: $scope.state.project}, 
function (request) {
+ModelService.save({}, {
+  
modelDescData:VdmUtil.filterNullValInObj($scope.state.modelSchema),
+  project: $scope.state.project
+}, function (request) {
 if(request.successful) {

[50/50] [abbrv] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-12-06 Thread lidong

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db1b826a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db1b826a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db1b826a

Branch: refs/heads/master-cdh5.7
Commit: db1b826ac6b6d086479dc044e88e58b7603c0a95
Parents: 469d9cc
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Wed Dec 7 12:30:13 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[26/50] [abbrv] kylin git commit: KYLIN-2245 refine CubeSegments

2016-12-06 Thread lidong

KYLIN-2245 refine CubeSegments

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ca53954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ca53954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ca53954

Branch: refs/heads/master-cdh5.7
Commit: 3ca53954415c62c2a91473c6d7bf831504b5e9db
Parents: 4a0ee79
Author: Cheng Wang 
Authored: Fri Dec 2 15:35:52 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 17:26:07 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java | 115 +++
 .../java/org/apache/kylin/cube/CubeManager.java |  10 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   8 +-
 .../apache/kylin/metadata/model/ISegment.java   |   6 +-
 .../apache/kylin/metadata/model/Segments.java   | 140 +++
 .../hbase/util/ExtendCubeToHybridCLI.java   |   4 +-
 .../kylin/tool/ExtendCubeToHybridCLI.java   |   4 +-
 7 files changed, 176 insertions(+), 111 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 061ab23..8b12c2e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,8 +18,6 @@
 
 package org.apache.kylin.cube;
 
-import java.util.ArrayList;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
@@ -34,6 +32,7 @@ import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
@@ -64,7 +63,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 cubeInstance.setName(cubeName);
 cubeInstance.setDescName(cubeDesc.getName());
 cubeInstance.setCreateTimeUTC(System.currentTimeMillis());
-cubeInstance.setSegments(new ArrayList());
+cubeInstance.setSegments(new Segments());
 cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
 cubeInstance.updateRandomUuid();
 
@@ -87,7 +86,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 
 @JsonManagedReference
 @JsonProperty("segments")
-private List segments = new ArrayList();
+private Segments segments = new Segments();
 
 @JsonProperty("create_time_utc")
 private long createTimeUTC;
@@ -97,39 +96,11 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 public List getBuildingSegments() {
-List buildingSegments = new ArrayList();
-if (null != segments) {
-for (CubeSegment segment : segments) {
-if (SegmentStatusEnum.NEW == segment.getStatus() || 
SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
-buildingSegments.add(segment);
-}
-}
-}
-
-return buildingSegments;
+return segments.getBuildingSegments();
 }
 
 public List getMergingSegments(CubeSegment mergedSegment) {
-LinkedList result = new LinkedList();
-if (mergedSegment == null)
-return result;
-
-for (CubeSegment seg : this.segments) {
-if (seg.getStatus() != SegmentStatusEnum.READY && seg.getStatus() 
!= SegmentStatusEnum.READY_PENDING)
-continue;
-
-if (seg == mergedSegment)
-continue;
-
-if (mergedSegment.sourceOffsetContains(seg)) {
-// make sure no holes
-if (result.size() > 0 && result.getLast().getSourceOffsetEnd() 
!= seg.getSourceOffsetStart())
-throw new IllegalStateException("Merging segments must not 
have holes between " + result.getLast() + " and " + seg);
-
-result.add(seg);
-}
-}
-return result;
+return segments.getMergingSegments(mergedSegment);
 }
 
 public CubeDesc getDescriptor() {
@@ -154,7 +125,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // in a temporary broken state, so that user can edit and fix it. Broken 
state is often due to
 // schema

[16/50] [abbrv] kylin git commit: KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally

2016-12-06 Thread lidong

 KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28ba1eae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28ba1eae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28ba1eae

Branch: refs/heads/master-cdh5.7
Commit: 28ba1eaeac8fd29ff3872f6d14604813a89b3a6e
Parents: 93bf0d0
Author: Hongbin Ma 
Authored: Thu Dec 1 11:35:34 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 11:46:00 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java   | 12 
 .../main/java/org/apache/kylin/cube/model/CubeDesc.java |  6 +-
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28ba1eae/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 766c04d..3c10826 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -79,7 +79,7 @@ abstract public class KylinConfigBase implements Serializable 
{
 protected KylinConfigBase(Properties props, boolean force) {
 this.properties = force ? props : BCC.check(props);
 }
-
+
 final protected String getOptional(String prop) {
 return getOptional(prop, null);
 }
@@ -161,7 +161,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public boolean isDevEnv() {
 return "DEV".equals(getOptional("kylin.env", "DEV"));
 }
-
+
 public String getDeployEnv() {
 return getOptional("kylin.env", "DEV");
 }
@@ -211,7 +211,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public Map getCubeCustomMeasureTypes() {
 return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
 }
-
+
 // 

 // DICTIONARY & SNAPSHOT
 // 

@@ -278,6 +278,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
 }
 
+public boolean isIgnoreCubeSignatureInconsistency() {
+return 
Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", 
"false"));
+}
+
 @Deprecated
 public int getCubeAggrGroupMaxSize() {
 return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", 
"12"));
@@ -298,7 +302,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public void setMaxBuildingSegments(int maxBuildingSegments) {
 setProperty("kylin.cube.max-building-segments", 
String.valueOf(maxBuildingSegments));
 }
-
+
 // 

 // JOB
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/28ba1eae/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 47063d0..327ce57 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -466,6 +466,11 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
  * @return
  */
 public boolean checkSignature() {
+if (this.getConfig().isIgnoreCubeSignatureInconsistency()) {
+logger.info("Skip checking cube signature");
+return true;
+}
+
 if (KylinVersion.getCurrentVersion().isCompatibleWith(new 
KylinVersion(getVersion())) && 
!KylinVersion.getCurrentVersion().isSignatureCompatibleWith(new 
KylinVersion(getVersion( {
 logger.info("checkSignature on {} is skipped as the its version is 
{} (not signature compatible but compatible) ", getName(), getVersion());
 return true;
@@ -1102,7 +1107,6 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 return newCubeDesc;
 }
 
-
 private Collection ensureOrder(Collection c) {
 TreeSet set = new TreeSet();
 for (Object o : c)

[05/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index d8158ec..7349b37 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,1349,US,JP,ANALYST,Beijing
-1,2012-08-28,Others,175750,0,13,23.8563,0,1927,JP,JP,ANALYST,Beijing
-2,2012-02-16,ABIN,148324,15,13,88.3418,0,1005,DE,FR,ADMIN,Shanghai
-3,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,1209,UK,UK,ANALYST,Beijing
-4,2012-10-22,Others,140746,100,11,83.454,0,1154,JP,CN,ADMIN,Shanghai
-5,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,1372,DE,UK,ADMIN,Shanghai
-6,2013-04-04,Others,963,0,13,88.5907,0,1648,US,JP,MODELER,Hongkong
-7,2012-04-11,Others,15687,0,15,88.194,0,1866,US,UK,ADMIN,Shanghai
-8,2013-01-19,ABIN,60606,3,13,77.9727,0,1936,FR,CN,MODELER,Hongkong
-9,2012-04-30,FP-non GTC,106246,0,14,52.295,0,1047,CN,FR,MODELER,Hongkong
-10,2013-02-03,Auction,45333,0,16,56.3584,0,1470,UK,JP,MODELER,Hongkong
-11,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,1203,US,UK,ANALYST,Beijing
-12,2013-07-30,Others,24760,0,16,25.077,0,1382,CN,JP,ADMIN,Shanghai
-13,2012-04-17,Auction,31519,0,16,72.7384,0,1069,CN,JP,ADMIN,Shanghai
-14,2013-04-30,FP-GTC,51582,0,15,75.82,0,1085,CN,DE,ANALYST,Beijing
-15,2013-03-23,FP-GTC,16509,0,15,8.6653,0,1559,US,FR,MODELER,Hongkong
-16,2013-04-01,ABIN,175750,3,16,64.2802,0,1507,JP,CN,MODELER,Hongkong
-17,2013-07-12,ABIN,50677,0,-99,24.5987,0,1938,UK,FR,ADMIN,Shanghai
-18,2012-04-23,FP-GTC,45333,0,5,72.6553,0,1863,CN,CN,MODELER,Hongkong
-19,2012-12-19,ABIN,3838,0,12,87.4142,0,1612,DE,JP,ADMIN,Shanghai
-20,2012-08-15,FP-GTC,62179,0,16,67.4238,0,1839,JP,US,MODELER,Hongkong
-21,2013-08-22,Auction,1357,0,5,18.36,0,1303,CN,JP,ANALYST,Beijing
-22,2013-08-23,ABIN,31519,0,5,67.6344,0,1877,FR,JP,ANALYST,Beijing
-23,2013-09-12,FP-GTC,223,0,14,71.7898,0,1848,US,CN,MODELER,Hongkong
-24,2012-03-14,FP-GTC,132939,0,15,26.1496,0,1815,JP,DE,MODELER,Hongkong
-25,2013-07-13,Others,67703,3,16,83.9661,0,1858,CN,CN,MODELER,Hongkong
-26,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,1387,CN,CN,ADMIN,Shanghai
-27,2012-08-30,Auction,62179,0,11,21.0496,0,1393,JP,FR,ANALYST,Beijing
-28,2013-01-10,FP-GTC,48027,0,13,23.4142,0,1757,FR,FR,MODELER,Hongkong
-29,2013-01-25,ABIN,145970,0,5,26.7842,0,1402,DE,CN,ANALYST,Beijing
-30,2012-07-05,Auction,48027,0,-99,46.732,0,1972,DE,UK,MODELER,Hongkong
-31,2012-11-07,FP-GTC,20213,0,-99,19.727,0,1873,UK,CN,ANALYST,Beijing
-32,2013-06-20,ABIN,100847,0,5,76.2783,0,1409,CN,DE,ADMIN,Shanghai
-33,2012-10-08,ABIN,26262,0,5,48.7447,0,1029,CN,FR,ADMIN,Shanghai
-34,2013-04-30,ABIN,87118,0,15,63.2187,0,1529,FR,DE,ANALYST,Beijing
-35,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,1329,JP,FR,ANALYST,Beijing
-36,2012-05-13,Others,164261,0,13,84.1246,0,1688,CN,GB,ADMIN,Shanghai
-37,2013-08-02,ABIN,31519,0,5,37.1504,0,1905,FR,DE,MODELER,Hongkong
-38,2012-04-03,Auction,164261,0,11,92.0974,0,1341,CN,UK,ADMIN,Shanghai
-39,2012-09-02,FP-GTC,31519,0,14,87.7829,0,1460,DE,UK,ANALYST,Beijing
-40,2013-10-05,ABIN,13987,0,16,74.0719,0,1381,CN,UK,MODELER,Hongkong
-41,2012-01-13,FP-GTC,48027,0,15,71.2637,0,1915,UK,CN,ADMIN,Shanghai
-42,2013-03-01,Others,13836,0,14,16.7288,0,1953,CN,UK,ADMIN,Shanghai
-43,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,1277,FR,CN,MODELER,Hongkong
-44,2012-02-07,Auction,45333,0,5,64.977,0,1040,FR,JP,MODELER,Hongkong
-45,2013-12-14,FP-non GTC,158798,0,16,72.4413,0,1500,DE,CN,MODELER,Hongkong
-46,2013-10-13,Auction,31519,0,5,79.3053,0,1816,CN,JP,MODELER,Hongkong
-47,2012-06-11,ABIN,4943,0,5,11.6942,0,1696,US,UK,ANALYST,Beijing
-48,2012-10-18,ABIN,80053,0,-99,54.0933,0,1604,FR,JP,MODELER,Hongkong
-49,2012-11-03,Auction,20886,0,5,9.8258,0,1764,US,CN,MODELER,Hongkong
-50,2012-01-15,Auction,44079,0,14,13.0371,0,1343,CN,US,MODELER,Hongkong
-51,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,1724,CN,GB,ADMIN,Shanghai
-52,2012-02-16,FP-GTC,314,0,13,49.1825,0,1334,JP,CN,ADMIN,Shanghai
-53,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,1603,JP,JP,ANALYST,Beijing
-54,2012-07-15,FP-non GTC,32876,0,-99,50.9634,0,1248,CN,UK,ANALYST,Beijing
-55,2012-07-24,FP-GTC,150265,15,16,57.5645,0,1653,CN,CN,ADMIN,Shanghai
-56,2012-10-06,Auction,145970,0,5,44.3091,0,1082,JP,DE,ADMIN,Shanghai
-57,2013-06-14,ABIN,80053,0,5,20.2388,0,1632,JP,UK,MODELER,Hongkong
-58,2013-10-10,FP-non GTC,15868,0,11,28.629,0,1523,US,FR,MODELER,Hongkong
-59,2012-09-30,ABIN,31673,0,12,75.8658,0,1719,CN,FR,ADMIN,Shanghai
-60,2012-04-02,FP-GTC,63861,0,5,25.8748,0,1311,JP,FR,ANALYST,Beijing

[36/50] [abbrv] kylin git commit: KYLIN-2248 TopN merge further optimization after KYLIN-1917

2016-12-06 Thread lidong

KYLIN-2248 TopN merge further optimization after KYLIN-1917


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59a30f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59a30f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59a30f66

Branch: refs/heads/master-cdh5.7
Commit: 59a30f66d47cc1838e6852405699fd7957bfac29
Parents: af429e5
Author: shaofengshi 
Authored: Sun Dec 4 09:39:45 2016 +0800
Committer: shaofengshi 
Committed: Mon Dec 5 17:42:34 2016 +0800

--
 .../apache/kylin/measure/topn/TopNCounter.java  | 47 ++--
 1 file changed, 13 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59a30f66/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 968e694..caf7961 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -26,11 +26,9 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import com.google.common.collect.Maps;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 
 /**
  * Modified from the StreamSummary.java in 
https://github.com/addthis/stream-lib
@@ -157,41 +155,22 @@ public class TopNCounter implements 
Iterable {
  * @return
  */
 public TopNCounter merge(TopNCounter another) {
-double m1 = 0.0, m2 = 0.0;
-if (this.size() >= this.capacity) {
-m1 = this.counterList.getLast().count;
-}
-
-if (another.size() >= another.capacity) {
-m2 = another.counterList.getLast().count;
-}
-
-Set duplicateItems = Sets.newHashSet();
-List notDuplicateItems = Lists.newArrayList();
-
-for (Map.Entry entry : this.counterMap.entrySet()) {
-T item = entry.getKey();
-Counter existing = another.counterMap.get(item);
-if (existing != null) {
-duplicateItems.add(item);
-} else {
-notDuplicateItems.add(item);
+boolean thisFull = this.size() >= this.capacity;
+boolean anotherFull = another.size() >= another.capacity;
+double m1 = thisFull ? this.counterList.getLast().count : 0.0;
+double m2 = anotherFull ? another.counterList.getLast().count : 0.0;
+
+if (anotherFull == true) {
+for (Counter entry : this.counterMap.values()) {
+entry.count += m2;
 }
 }
 
-for (T item : duplicateItems) {
-this.offer(item, another.counterMap.get(item).count);
-}
-
-for (T item : notDuplicateItems) {
-this.offer(item, m2);
-}
-
 for (Map.Entry entry : another.counterMap.entrySet()) {
-T item = entry.getKey();
-if (duplicateItems.contains(item) == false) {
-double counter = entry.getValue().count;
-this.offer(item, counter + m1);
+if (this.counterMap.containsKey(entry.getKey())) {
+this.offer(entry.getValue().getItem(), (entry.getValue().count 
- m2));
+} else {
+this.offer(entry.getValue().getItem(), entry.getValue().count 
+ m1);
 }
 }

[02/50] [abbrv] kylin git commit: KYLIN-2195 support backward-compatibility properties in get-properties.sh

2016-12-06 Thread lidong

KYLIN-2195 support backward-compatibility properties in get-properties.sh


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1448e57
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1448e57
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1448e57

Branch: refs/heads/master-cdh5.7
Commit: b1448e5789ce65d5b94ed246df01fa4c269515b2
Parents: a2ecf18
Author: lidongsjtu 
Authored: Mon Nov 28 23:37:41 2016 +0800
Committer: lidongsjtu 
Committed: Mon Nov 28 23:37:41 2016 +0800

--
 build/bin/get-properties.sh |  4 +-
 .../org/apache/kylin/tool/KylinConfigCLI.java   | 42 
 2 files changed, 45 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b1448e57/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 1a086ea..170442d 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -25,5 +25,7 @@ then
 exit -1
 fi
 
-result=`cat ${KYLIN_HOME}/conf/kylin.properties | grep -w "^$1" | grep -v '^#' 
| awk -F= '{ n = index($0,"="); print substr($0,n+1)}' | cut -c 1- |tail -1`
+job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 
2>/dev/null`
 echo "$result"

http://git-wip-us.apache.org/repos/asf/kylin/blob/b1448e57/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
new file mode 100644
index 000..e1a5b99
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool;
+
+import java.util.Properties;
+
+import org.apache.kylin.common.BackwardCompatibilityConfig;
+import org.apache.kylin.common.KylinConfig;
+
+public class KylinConfigCLI {
+public static void main(String[] args) {
+if (args.length != 1) {
+System.err.println("Usage: KylinConfigCLI conf_name");
+System.err.println("Example: KylinConfigCLI kylin.server.mode");
+System.exit(1);
+}
+
+Properties config = KylinConfig.getKylinProperties();
+BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig();
+String value = config.getProperty(bcc.check(args[0]));
+if (value == null) {
+value = "";
+}
+System.out.println(value);
+}
+}

[46/50] [abbrv] kylin git commit: KYLIN-2252, Enhance project/model/cube name check

2016-12-06 Thread lidong

KYLIN-2252, Enhance project/model/cube name check


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e787165
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e787165
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e787165

Branch: refs/heads/master-cdh5.7
Commit: 1e787165edee713a59db8627e70f79edabfd7d9d
Parents: ea13af4
Author: Billy Liu 
Authored: Wed Dec 7 09:42:58 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 09:42:58 2016 +0800

--
 .../org/apache/kylin/rest/controller/CubeController.java  | 10 ++
 .../apache/kylin/rest/controller/CubeDescController.java  |  4 ++--
 .../org/apache/kylin/rest/controller/ModelController.java | 10 ++
 .../apache/kylin/rest/controller/ProjectController.java   | 10 +-
 4 files changed, 31 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 3846d28..f537231 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -83,6 +83,8 @@ import com.google.common.collect.Maps;
 public class CubeController extends BasicController {
 private static final Logger logger = 
LoggerFactory.getLogger(CubeController.class);
 
+private static final char[] VALID_CUBENAME = 
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
+
 @Autowired
 private CubeService cubeService;
 
@@ -343,6 +345,10 @@ public class CubeController extends BasicController {
 if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
 throw new BadRequestException("Broken cube can't be cloned");
 }
+if (!StringUtils.containsOnly(newCubeName, VALID_CUBENAME)) {
+logger.info("Invalid Cube name {}, only letters, numbers and 
underline supported.", newCubeName);
+throw new BadRequestException("Invalid Cube name, only letters, 
numbers and underline supported.");
+}
 
 CubeDesc cubeDesc = cube.getDescriptor();
 CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
@@ -421,6 +427,10 @@ public class CubeController extends BasicController {
 logger.info("Cube name should not be empty.");
 throw new BadRequestException("Cube name should not be empty.");
 }
+if (!StringUtils.containsOnly(name, VALID_CUBENAME)) {
+logger.info("Invalid Cube name {}, only letters, numbers and 
underline supported.", name);
+throw new BadRequestException("Invalid Cube name, only letters, 
numbers and underline supported.");
+}
 
 try {
 desc.setUuid(UUID.randomUUID().toString());

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
index 5a8eeec..0c8f487 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -44,8 +44,8 @@ public class CubeDescController extends BasicController {
 /**
  * Get detail information of the "Cube ID"
  * 
- * @param cubeDescName
- *Cube ID
+ * @param cubeName
+ *Cube Name
  * @return
  * @throws IOException
  */

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
index 5f6a91b..df9ecfb 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -61,6 +61,8 @@ import com.fasterxml.jackson.databind.JsonMappingException;
 public class ModelController extends BasicController {
 private static final Logger logger =

[41/50] [abbrv] kylin git commit: minor, generate item_count value for sample data

2016-12-06 Thread lidong

minor, generate item_count value for sample data

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2aaf270
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2aaf270
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2aaf270

Branch: refs/heads/master-cdh5.7
Commit: d2aaf2705a9035be9dbef8205fc9a1358b4316c2
Parents: 9968c22
Author: Yiming Liu 
Authored: Fri Dec 2 16:39:00 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 6 13:50:28 2016 +0800

--
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 1 file changed, 1 insertions(+), 1 deletions(-)
--

[23/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global Dictionary

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
--
diff --git 
a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
 
b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
index caa8754..949599d 100644
--- 
a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
+++ 
b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
@@ -1,402 +1,402 @@
-2013-03-31,48028,0,\N,\N,\N,Auction,12,184.21,1001,1,Ebay
-2013-11-12,164262,0,\N,\N,\N,Others,5,172.03,1002,1,Ebay
-2013-04-06,82494,15,BookMagazines,NULL,Comic 
Books,Auction,14,66.6,1003,1,Ebay
-2013-05-17,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,1004,1,Ebay
-2013-05-20,152801,0,Jewelry & 
Watches,NULL,Earrings,FP-GTC,5,132.33,1005,1,Ebay
-2013-06-16,43398,0,Home & Garden,NULL,Cheese & 
Crackers,FP-GTC,13,7.12,1006,1,Ebay
-2013-06-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & 
Kits,Auction,14,204.28,1007,1,Ebay
-2013-03-22,158666,15,ToyHobbies,Action Figures,Anime & 
Manga,Auction,13,35.72,1008,1,Ebay
-2013-03-10,12688,0,eBay Premier,Books & Manuscripts,Books: 
Other,Auction,12,4.13,1009,1,Ebay
-2013-11-01,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed 
Items,FP-GTC,5,27.48,1010,1,Ebay
-2013-06-16,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & 
Cleaning,FP-GTC,14,9.26,1011,1,Ebay
-2013-09-12,80287,0,Computers/Tablets & Networking,Software,Office & 
Business,Auction,12,3.18,1012,1,Ebay
-2013-09-28,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck 
Parts,Others,13,3.18,1013,1,Ebay
-2013-06-15,87118,0,Sporting Goods,Outdoor 
Sports,Paintball,ABIN,14,377.94,1014,1,Ebay
-2013-03-14,25147,0,Sports MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Baseball-MLB,Auction,12,146.33,1015,1,Ebay
-2013-09-01,170302,15,Crafts,Embroidery,Design 
CDs,FP-GTC,5,51.23,1016,1,Ebay
-2013-05-29,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage 
Farm Equip,FP-non GTC,13,72.65,1017,1,Ebay
-2013-05-31,132939,0,Jewelry & Watches,Fashion 
Jewelry,Other,Auction,13,66.6,1018,1,Ebay
-2013-03-18,113593,15,Phones,Mobile Phones,Mobile 
Phones,Auction,12,9.26,1019,1,Ebay
-2013-07-19,34273,100,eBay Motors,Parts & 
Accessories,Motorcycle,Auction,14,583.44,1020,1,Ebay
-2013-06-23,106340,15,Home & Garden,Gardening,Hand 
Tools,FP-GTC,14,638.72,1021,1,Ebay
-2013-05-20,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,1022,1,Ebay
-2013-05-17,24760,0,Sports MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Hockey-NHL,FP-GTC,12,319.79,1023,1,Ebay
-2013-03-11,37831,0,Collectibles,Advertising,Merchandise & 
Memorabilia,Auction,12,20.35,1024,1,Ebay
-2013-01-30,1120,3,Books,First Editions,Other,FP-non 
GTC,5,223.63,1025,1,Ebay
-2013-01-26,43972,100,eBay Motors,Parts & Accessories,ATV 
Parts,FP-GTC,13,204.28,1026,1,Ebay
-2013-03-22,166013,15,Computers,Computer Components & Parts,Video Capture & TV 
Tuner Cards,Auction,14,5.48,1027,1,Ebay
-2013-07-23,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,1028,1,Ebay
-2013-07-27,103178,15,ClothinShoeAccessories,Women's Bags,Women's 
Bags,FP-GTC,5,21.72,1029,1,Ebay
-2013-10-29,2023,0,Sporting Goods,Team 
Sports,Basketball,ABIN,12,3.18,1030,1,Ebay
-2013-10-08,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car 
Video,FP-GTC,11,491.32,1031,1,Ebay
-2013-04-26,15868,0,Real Estate,Land,Land,Auction,14,448.8,1032,1,Ebay
-2013-01-01,32876,0,Home & Garden,Home Improvement,Plumbing & 
Fixtures,Auction,13,415.73,1033,1,Ebay
-2013-01-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic 
Apparel,Auction,13,377.94,1034,1,Ebay
-2013-05-27,33038,15,Musical Instruments,Instruments,Guitars 
(Electric),FP-GTC,14,146.33,1035,1,Ebay
-2013-11-11,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: 
RacinNASCAR,FP-GTC,5,7.12,1036,1,Ebay
-2013-03-08,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & 
Conditioning,Auction,13,42.99,1037,1,Ebay
-2013-03-25,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic 
Apparel,Auction,13,12.85,1038,1,Ebay
-2013-08-20,15115,0,Video Games & Consoles,Video Games,Video 
Games,FP-GTC,13,55.89,1039,1,Ebay
-2013-05-17,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm 
Bracelets,FP-GTC,14,73.26,1040,1,Ebay
-2013-06-05,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & 
Vans,Auction,11,112.56,1041,1,Ebay
-2013-10-08,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio 
Accessories,FP-non GTC,11,3.49,1042,1,Ebay
-2013-08-14,121153,0,Baby,Nursery Decor,Night 
Lights,Auction,13,184.21,1043,1,Ebay
-2013-08-14,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & 
Laser Detectors,Auction,13,157.14,1044,1,Ebay

[25/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global Dictionary

2016-12-06 Thread lidong

KYLIN-2192 More Robust Global Dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a0ee798
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a0ee798
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a0ee798

Branch: refs/heads/master-cdh5.7
Commit: 4a0ee7989d5f8272592b980fce3f5716ca40d4c1
Parents: e562aaf
Author: sunyerui 
Authored: Mon Nov 21 21:26:34 2016 +0800
Committer: gaodayue 
Committed: Fri Dec 2 13:33:59 2016 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   |  12 +-
 .../apache/kylin/common/KylinConfigBase.java|   8 +
 .../apache/kylin/common/util/Dictionary.java|   2 +-
 .../model/validation/rule/DictionaryRule.java   |  78 +-
 .../validation/rule/DictionaryRuleTest.java |  28 +-
 .../apache/kylin/dict/AppendTrieDictionary.java | 285 +--
 .../kylin/dict/AppendTrieDictionaryChecker.java | 102 +++
 .../org/apache/kylin/dict/CachedTreeMap.java| 260 +++---
 .../kylin/dict/GlobalDictionaryBuilder.java |  36 +-
 .../kylin/dict/AppendTrieDictionaryTest.java| 150 +++-
 .../apache/kylin/dict/CachedTreeMapTest.java| 320 +---
 .../kylin/measure/bitmap/BitmapCounterTest.java |   6 +-
 ...t_kylin_cube_without_slr_left_join_desc.json |  16 +-
 .../localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv  | 804 +--
 .../flatten_data_for_without_slr_left_join.csv  | 804 +--
 .../test_kylin_inner_join_model_desc.json   |   3 +-
 .../test_kylin_inner_join_view_model_desc.json  |   3 +-
 .../test_kylin_left_join_model_desc.json|   3 +-
 .../test_kylin_left_join_view_model_desc.json   |   3 +-
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |   8 +-
 .../source/hive/ITHiveTableReaderTest.java  |   2 +-
 .../query/sql_distinct_precisely/query00.sql|   2 +-
 .../query/sql_distinct_precisely/query01.sql|   2 +-
 .../query/sql_distinct_precisely/query02.sql|   2 +-
 .../query/sql_distinct_precisely/query03.sql|   3 +-
 .../query/sql_distinct_precisely/query04.sql|   3 +-
 .../query/sql_distinct_precisely/query05.sql|  25 -
 .../query/sql_distinct_precisely/query06.sql|  26 -
 .../query/sql_distinct_precisely/query07.sql|  24 -
 29 files changed, 1737 insertions(+), 1283 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 8068fd1..677b713 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -403,13 +403,13 @@ public class FactTableGenerator {
 }
 
 private String createRandomCell(ColumnDesc cDesc) {
-String type = cDesc.getTypeName();
-String s = type.toLowerCase();
-if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
+DataType type =cDesc.getType();
+String s = type.getName();
+if (s.equals("char") || s.equals("varchar")) {
 StringBuilder sb = new StringBuilder();
-for (int i = 0; i < 2; i++) {
-sb.append((char) ('a' + r.nextInt(10)));// there are 10*10
-// possible strings
+int len = Math.min(type.getPrecision(), 3);
+for (int i = 0; i < len; i++) {
+sb.append((char) ('a' + r.nextInt(10)));  // cardinality at 
most 10x10x10
 }
 return sb.toString();
 } else if (s.equals("bigint") || s.equals("int") || 
s.equals("tinyint") || s.equals("smallint")) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3c10826..f46c185 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -232,6 +232,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "1000"));
 }
 
+public int getAppendDictMaxVersions() {
+return 
Integer.parseInt(getOptional("kylin.dictionary.append-max-versions", "3"));
+}
+
+public int getAppendDictVersionTTL() {
+return

[48/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

[37/50] [abbrv] kylin git commit: KYLIN-2245 slim Segments in CubeMananger

2016-12-06 Thread lidong

KYLIN-2245 slim Segments in CubeMananger

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a441c3f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a441c3f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a441c3f

Branch: refs/heads/master-cdh5.7
Commit: 0a441c3fa30aee0a06cfc6301f7fbfa412103179
Parents: 59a30f6
Author: Cheng Wang 
Authored: Mon Dec 5 16:17:31 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 20:20:30 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |  14 ++
 .../java/org/apache/kylin/cube/CubeManager.java | 165 +--
 .../apache/kylin/metadata/model/ISegment.java   |   5 +-
 .../apache/kylin/metadata/model/Segments.java   | 206 ++-
 4 files changed, 221 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 8b12c2e..ecbb437 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.cube;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Set;
 
@@ -25,6 +26,7 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -361,6 +363,18 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return this.getDescriptor().getAutoMergeTimeRanges() != null && 
this.getDescriptor().getAutoMergeTimeRanges().length > 0;
 }
 
+public Pair autoMergeCubeSegments() throws IOException {
+return segments.autoMergeCubeSegments(needAutoMerge(), getName(), 
getDescriptor().getAutoMergeTimeRanges());
+}
+
+public Segments calculateToBeSegments(CubeSegment newSegment) {
+return segments.calculateToBeSegments(newSegment, 
getModel().getPartitionDesc().isPartitioned());
+}
+
+public Pair 
findMergeOffsetsByDateRange(Segments segs, long startDate, long 
endDate, long skipSegDateRangeCap) {
+return this.segments.findMergeOffsetsByDateRange(segs, startDate, 
endDate, skipSegDateRangeCap);
+}
+
 public CubeSegment getLastSegment() {
 List existing = getSegments();
 if (existing.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4ba29af..296a4e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -520,7 +519,7 @@ public class CubeManager implements IRealizationProvider {
 if (isOffsetsOn) {
 // offset cube, merge by date range?
 if (startOffset == endOffset) {
-Pair pair = 
findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
+Pair pair = 
cube.findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
 if (pair == null)
 throw new IllegalArgumentException("Find no segments to 
merge by date range " + startDate + "-" + endDate + " for cube " + cube);
 startOffset = pair.getFirst().getSourceOffsetStart();
@@ -580,32 +579,6 @@ public class CubeManager implements IRealizationProvider {
 return newSegment;
 }
 
-private Pair 
findMergeOffsetsByDateRange(List segments, long startDate, long 
endDate, long skipSegDateRangeCap) {
-// must be offset cube
-

[19/50] [abbrv] kylin git commit: KYLIN-2180 minor, get owenr project in CubeDesc.init()

2016-12-06 Thread lidong

KYLIN-2180 minor, get owenr project in CubeDesc.init()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9fc8f5e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9fc8f5e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9fc8f5e2

Branch: refs/heads/master-cdh5.7
Commit: 9fc8f5e207b5111fffaec3335efe0c8947353cfa
Parents: 1a5295b
Author: Li Yang 
Authored: Fri Dec 2 11:39:34 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 11:39:34 2016 +0800

--
 .../main/java/org/apache/kylin/cube/model/CubeDesc.java  | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9fc8f5e2/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 327ce57..853571c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -64,6 +64,9 @@ import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -519,11 +522,16 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public void init(KylinConfig config) {
 this.errors.clear();
-this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
 
 checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank");
 checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc(%s) has 
blank modelName", name);
 
+// note CubeDesc.name == CubeInstance.name
+List ownerPrj = 
ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name);
+logger.info("CubeDesc '" + name + "' is owned by " + ownerPrj);
+
+this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
+
 this.model = 
MetadataManager.getInstance(config).getDataModelDesc(modelName);
 checkNotNull(this.model, "DateModelDesc(%s) not found", modelName);
 
@@ -823,6 +831,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 return col;
 }
 
+@SuppressWarnings("deprecation")
 private void initMeasureColumns() {
 if (measures == null || measures.isEmpty()) {
 return;

[44/50] [abbrv] kylin git commit: KYLIN-1971 ColumnDesc.equals() bug fix

2016-12-06 Thread lidong

KYLIN-1971 ColumnDesc.equals() bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a53eafaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a53eafaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a53eafaa

Branch: refs/heads/master-cdh5.7
Commit: a53eafaa30d5bb8bf5b52404bb6bbc1dcd9673e8
Parents: b5c0588
Author: Li Yang 
Authored: Tue Dec 6 18:39:03 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 6 18:39:03 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a53eafaa/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 7d9133d..403eaaf 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -194,7 +194,7 @@ public class ColumnDesc implements Serializable {
 if (table == null) {
 if (other.table != null)
 return false;
-} else if (!table.equals(other.table))
+} else if (!table.getIdentity().equals(other.table.getIdentity()))
 return false;
 
 if (datatype == null) {

[17/50] [abbrv] kylin git commit: refine mapper and reducer log

2016-12-06 Thread lidong

refine mapper and reducer log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58224921
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58224921
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58224921

Branch: refs/heads/master-cdh5.7
Commit: 58224921d896e4479f5d034d43c044aacaf14200
Parents: 28ba1ea
Author: Hongbin Ma 
Authored: Thu Dec 1 18:15:46 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 18:15:46 2016 +0800

--
 .../java/org/apache/kylin/engine/mr/KylinMapper.java   | 12 +---
 .../java/org/apache/kylin/engine/mr/KylinReducer.java  | 13 ++---
 .../apache/kylin/engine/mr/steps/CuboidReducer.java| 11 ---
 .../kylin/engine/mr/steps/HiveToBaseCuboidMapper.java  |  6 --
 .../kylin/engine/mr/steps/InMemCuboidMapper.java   |  7 +--
 .../kylin/engine/mr/steps/InMemCuboidReducer.java  | 10 +-
 .../apache/kylin/engine/mr/steps/NDCuboidMapper.java   | 10 --
 7 files changed, 33 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index a01f7a2..2b564e9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory;
 public class KylinMapper extends 
Mapper {
 private static final Logger logger = 
LoggerFactory.getLogger(KylinMapper.class);
 
+protected int mapCounter = 0;
+
 protected void bindCurrentConfiguration(Configuration conf) {
 logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 HadoopUtil.setCurrentConfiguration(conf);
@@ -38,6 +41,9 @@ public class KylinMapper 
extends Mapper.Context context) throws IOException, InterruptedException {
 try {
+if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 
0) {
+logger.info("Accepting Mapper Key with ordinal: " + 
mapCounter);
+}
 doMap(key, value, context);
 } catch (IOException ex) { // KYLIN-2170
 logger.error("", ex);
@@ -53,11 +59,11 @@ public class KylinMapper 
extends Mapper.Context context) throws IOException, InterruptedException {
 super.map(key, value, context);
 }
-
+
 @Override
 final protected void cleanup(Mapper.Context context) throws IOException, InterruptedException {
 try {
@@ -76,7 +82,7 @@ public class KylinMapper 
extends Mapper.Context 
context) throws IOException, InterruptedException {
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index 2b63ce0..cb2d7a7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -29,7 +30,9 @@ import org.slf4j.LoggerFactory;
  */
 public class KylinReducer extends 
Reducer {
 private static final Logger logger =

[30/50] [abbrv] kylin git commit: minor: normalize scientific decimal values

2016-12-06 Thread lidong

minor: normalize scientific decimal values


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fe9efa86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fe9efa86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fe9efa86

Branch: refs/heads/master-cdh5.7
Commit: fe9efa86cd34da31ec41dc176140ee4ee2f49fed
Parents: eb686a9
Author: Hongbin Ma 
Authored: Sun Dec 4 21:26:18 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Dec 4 21:26:18 2016 +0800

--
 .../java/org/apache/kylin/metadata/tuple/Tuple.java | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fe9efa86/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index 721a719..d51ae9e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -115,6 +115,8 @@ public class Tuple implements ITuple {
 // BigDecimal during cube build for best precision
 if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
 fieldValue = ((BigDecimal) fieldValue).doubleValue();
+} else if ("decimal".equals(dataType) && fieldValue instanceof 
BigDecimal) {
+fieldValue = normalizeDecimal((BigDecimal) fieldValue);
 } else if ("integer".equals(dataType) && fieldValue instanceof Number) 
{
 fieldValue = ((Number) fieldValue).intValue();
 } else if ("smallint".equals(dataType) && fieldValue instanceof 
Number) {
@@ -145,6 +147,14 @@ public class Tuple implements ITuple {
 return o;
 }
 
+private static BigDecimal normalizeDecimal(BigDecimal input) {
+if (input.scale() < 0) {
+return input.setScale(0);
+} else {
+return input;
+}
+}
+
 public boolean hasColumn(TblColRef column) {
 return info.hasColumn(column);
 }
@@ -199,7 +209,7 @@ public class Tuple implements ITuple {
 case "double":
 return Double.valueOf(strValue);
 case "decimal":
-return new BigDecimal(strValue);
+return normalizeDecimal(new BigDecimal(strValue));
 case "float":
 return Float.valueOf(strValue);
 case "boolean":

[29/50] [abbrv] kylin git commit: KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization

2016-12-06 Thread lidong

KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb686a94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb686a94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb686a94

Branch: refs/heads/master-cdh5.7
Commit: eb686a9428d930a60ec87883a2c253fb07f80a72
Parents: 0fd1ed6
Author: Li Yang 
Authored: Fri Dec 2 18:34:29 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 18:34:29 2016 +0800

--
 .../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb686a94/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 1e836b7..f6e687b 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -151,17 +151,21 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 BytesUtil.writeUTFString(name, buffer);
+buffer.put((byte) (isReversed ? 1 : 0));
 }
 
 @Override
 public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 this.name = BytesUtil.readUTFString(buffer);
+this.isReversed = buffer.get() != 0;
 this.initMethod();
 }
 
 @Override
 public String toString() {
 StringBuilder sb = new StringBuilder();
+if (isReversed)
+sb.append("NOT ");
 sb.append(name);
 sb.append("(");
 for (int i = 0; i < methodParams.size(); i++) {

[03/50] [abbrv] kylin git commit: minor, fix typo

2016-12-06 Thread lidong

minor, fix typo


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3d62e9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3d62e9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3d62e9c

Branch: refs/heads/master-cdh5.7
Commit: c3d62e9c5e6c39771bec1bbe2660130ff2033ef3
Parents: b1448e5
Author: lidongsjtu 
Authored: Mon Nov 28 23:48:46 2016 +0800
Committer: lidongsjtu 
Committed: Mon Nov 28 23:48:46 2016 +0800

--
 build/smoke-test/testDiag.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c3d62e9c/build/smoke-test/testDiag.py
--
diff --git a/build/smoke-test/testDiag.py b/build/smoke-test/testDiag.py
index cc932da..588864b 100644
--- a/build/smoke-test/testDiag.py
+++ b/build/smoke-test/testDiag.py
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# This is python unittest used in smoke-test.sh, aim to testing query via rest 
APIs.
+# This is python unittest used in smoke-test.sh, aim to testing diagnosis via 
rest APIs.
 
 import unittest
 import requests
@@ -40,5 +40,5 @@ class testDiag(unittest.TestCase):
 
 
 if __name__ == '__main__':
-print 'Test Diagnogis for Kylin sample.'
+print 'Test Diagnosis for Kylin sample.'
 unittest.main()

[45/50] [abbrv] kylin git commit: KYLIN-2251: fix httpcore dependency conflict in JDBC Driver

2016-12-06 Thread lidong

KYLIN-2251: fix httpcore dependency conflict in JDBC Driver


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea13af4d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea13af4d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea13af4d

Branch: refs/heads/master-cdh5.7
Commit: ea13af4d8aa6a5b25fc328ee1adaf2b454eddea0
Parents: a53eafa
Author: Billy Liu 
Authored: Wed Dec 7 09:01:52 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 09:01:52 2016 +0800

--
 jdbc/pom.xml | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea13af4d/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 4f42913..22d8eca 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -39,14 +39,14 @@
 
 
 
-org.apache.calcite.avatica
-avatica
-
-
 org.apache.httpcomponents
 httpclient
 
 
+org.apache.calcite.avatica
+avatica
+
+
 junit
 junit
 test

[38/50] [abbrv] kylin git commit: KYLIN-2245 code review

2016-12-06 Thread lidong

KYLIN-2245 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fc1e11aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fc1e11aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fc1e11aa

Branch: refs/heads/master-cdh5.7
Commit: fc1e11aa8a2d20add72e382396787d9cb09771da
Parents: 0a441c3
Author: Yang Li 
Authored: Mon Dec 5 21:20:21 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:20:21 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeInstance.java| 4 
 .../src/main/java/org/apache/kylin/cube/CubeManager.java | 2 +-
 .../main/java/org/apache/kylin/metadata/model/ISegment.java  | 4 ++--
 .../main/java/org/apache/kylin/metadata/model/Segments.java  | 8 
 4 files changed, 7 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index ecbb437..a3665f5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -371,10 +371,6 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return segments.calculateToBeSegments(newSegment, 
getModel().getPartitionDesc().isPartitioned());
 }
 
-public Pair 
findMergeOffsetsByDateRange(Segments segs, long startDate, long 
endDate, long skipSegDateRangeCap) {
-return this.segments.findMergeOffsetsByDateRange(segs, startDate, 
endDate, skipSegDateRangeCap);
-}
-
 public CubeSegment getLastSegment() {
 List existing = getSegments();
 if (existing.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 296a4e7..0c80c07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -519,7 +519,7 @@ public class CubeManager implements IRealizationProvider {
 if (isOffsetsOn) {
 // offset cube, merge by date range?
 if (startOffset == endOffset) {
-Pair pair = 
cube.findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
+Pair pair = 
cube.getSegments(SegmentStatusEnum.READY).findMergeOffsetsByDateRange(startDate,
 endDate, Long.MAX_VALUE);
 if (pair == null)
 throw new IllegalArgumentException("Find no segments to 
merge by date range " + startDate + "-" + endDate + " for cube " + cube);
 startOffset = pair.getFirst().getSourceOffsetStart();

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index 9d26927..f006613 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -26,6 +26,8 @@ public interface ISegment {
 
 public long getDateRangeEnd();
 
+public boolean isSourceOffsetsOn();
+
 public long getSourceOffsetStart();
 
 public long getSourceOffsetEnd();
@@ -35,6 +37,4 @@ public interface ISegment {
 public SegmentStatusEnum getStatus();
 
 public long getLastBuildTime();
-
-public boolean isSourceOffsetsOn();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index bc115cc..5198dc7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -187,8 +187,8 @@ public class Segments extends 
ArrayList {

[06/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
new file mode 100644
index 000..253afbf
--- /dev/null
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
@@ -0,0 +1,1 @@
+1000,1,4,FR,N/A
+1001,0,1,DE,N/A
+1002,0,1,IT,N/A
+1003,4,2,DE,N/A
+1004,2,3,JP,N/A
+1005,0,1,CN,N/A
+1006,2,3,JP,N/A
+1007,0,2,GB,N/A
+1008,5,3,US,N/A
+1009,1,3,US,N/A
+1010,1,1,IT,N/A
+1011,2,4,FR,N/A
+1012,3,1,CN,N/A
+1013,1,1,FR,N/A
+1014,5,3,CN,N/A
+1015,3,5,IT,N/A
+1016,4,5,JP,N/A
+1017,3,4,US,N/A
+1018,0,5,CN,N/A
+1019,4,4,GB,N/A
+1020,4,4,IT,N/A
+1021,1,2,FR,N/A
+1022,2,2,DE,N/A
+1023,0,1,GB,N/A
+1024,3,4,IT,N/A
+1025,4,4,JP,N/A
+1026,5,4,IT,N/A
+1027,2,4,FR,N/A
+1028,2,2,GB,N/A
+1029,1,1,GB,N/A
+1030,5,5,CN,N/A
+1031,4,2,CN,N/A
+1032,1,3,CN,N/A
+1033,2,1,JP,N/A
+1034,2,1,JP,N/A
+1035,4,2,US,N/A
+1036,5,4,DE,N/A
+1037,2,4,DE,N/A
+1038,1,3,US,N/A
+1039,3,5,US,N/A
+1040,2,3,GB,N/A
+1041,1,5,IT,N/A
+1042,5,5,CN,N/A
+1043,5,1,IT,N/A
+1044,3,4,IT,N/A
+1045,1,3,US,N/A
+1046,3,3,JP,N/A
+1047,2,2,FR,N/A
+1048,5,4,JP,N/A
+1049,3,1,JP,N/A
+1050,3,5,JP,N/A
+1051,5,1,US,N/A
+1052,3,5,DE,N/A
+1053,1,3,JP,N/A
+1054,5,1,GB,N/A
+1055,2,1,GB,N/A
+1056,2,3,CN,N/A
+1057,1,5,US,N/A
+1058,2,2,CN,N/A
+1059,2,4,FR,N/A
+1060,0,3,FR,N/A
+1061,0,5,FR,N/A
+1062,1,1,US,N/A
+1063,4,5,DE,N/A
+1064,2,5,GB,N/A
+1065,0,3,GB,N/A
+1066,1,1,IT,N/A
+1067,0,2,GB,N/A
+1068,4,1,IT,N/A
+1069,0,4,FR,N/A
+1070,3,4,CN,N/A
+1071,3,5,GB,N/A
+1072,4,3,CN,N/A
+1073,0,4,IT,N/A
+1074,5,3,GB,N/A
+1075,0,4,JP,N/A
+1076,2,5,JP,N/A
+1077,0,3,CN,N/A
+1078,4,3,GB,N/A
+1079,4,5,US,N/A
+1080,4,2,JP,N/A
+1081,5,2,CN,N/A
+1082,4,1,JP,N/A
+1083,0,4,GB,N/A
+1084,1,4,DE,N/A
+1085,4,4,FR,N/A
+1086,2,5,JP,N/A
+1087,0,1,IT,N/A
+1088,0,2,FR,N/A
+1089,3,2,DE,N/A
+1090,5,2,IT,N/A
+1091,0,3,GB,N/A
+1092,1,4,DE,N/A
+1093,2,1,IT,N/A
+1094,4,3,JP,N/A
+1095,1,1,DE,N/A
+1096,0,5,JP,N/A
+1097,3,3,CN,N/A
+1098,5,3,JP,N/A
+1099,3,1,GB,N/A
+1100,5,1,CN,N/A
+1101,2,3,JP,N/A
+1102,5,1,IT,N/A
+1103,5,2,GB,N/A
+1104,3,5,GB,N/A
+1105,3,1,DE,N/A
+1106,3,5,GB,N/A
+1107,5,3,DE,N/A
+1108,4,1,CN,N/A
+1109,4,3,GB,N/A
+1110,4,1,FR,N/A
+1111,3,5,FR,N/A
+1112,1,1,GB,N/A
+1113,5,4,FR,N/A
+1114,2,2,DE,N/A
+1115,4,3,GB,N/A
+1116,0,3,FR,N/A
+1117,1,4,CN,N/A
+1118,4,4,IT,N/A
+1119,0,2,CN,N/A
+1120,5,2,DE,N/A
+1121,0,2,GB,N/A
+1122,4,3,CN,N/A
+1123,1,4,FR,N/A
+1124,0,2,CN,N/A
+1125,4,5,US,N/A
+1126,0,4,GB,N/A
+1127,4,3,CN,N/A
+1128,4,3,GB,N/A
+1129,1,4,JP,N/A
+1130,2,2,JP,N/A
+1131,0,3,IT,N/A
+1132,4,1,GB,N/A
+1133,1,5,US,N/A
+1134,3,3,DE,N/A
+1135,3,2,IT,N/A
+1136,2,2,JP,N/A
+1137,0,3,CN,N/A
+1138,3,1,FR,N/A
+1139,2,3,FR,N/A
+1140,3,1,IT,N/A
+1141,0,3,US,N/A
+1142,3,4,FR,N/A
+1143,4,5,DE,N/A
+1144,3,3,JP,N/A
+1145,5,5,DE,N/A
+1146,2,5,FR,N/A
+1147,0,5,IT,N/A
+1148,2,5,US,N/A
+1149,0,3,CN,N/A
+1150,2,2,GB,N/A
+1151,5,5,CN,N/A
+1152,2,2,IT,N/A
+1153,3,2,FR,N/A
+1154,5,4,FR,N/A
+1155,3,5,FR,N/A
+1156,5,2,GB,N/A
+1157,1,2,IT,N/A
+1158,3,3,JP,N/A
+1159,5,1,IT,N/A
+1160,3,4,CN,N/A
+1161,2,5,IT,N/A
+1162,4,5,CN,N/A
+1163,3,5,FR,N/A
+1164,1,2,FR,N/A
+1165,0,3,GB,N/A
+1166,5,4,CN,N/A
+1167,4,5,IT,N/A
+1168,3,2,IT,N/A
+1169,3,2,DE,N/A
+1170,2,2,DE,N/A
+1171,3,1,IT,N/A
+1172,3,1,IT,N/A
+1173,3,5,CN,N/A
+1174,5,2,US,N/A
+1175,2,3,FR,N/A
+1176,5,5,JP,N/A
+1177,1,4,DE,N/A
+1178,4,5,GB,N/A
+1179,5,3,FR,N/A
+1180,2,3,GB,N/A
+1181,0,3,IT,N/A
+1182,3,2,JP,N/A
+1183,5,5,IT,N/A
+1184,4,1,IT,N/A
+1185,3,2,IT,N/A
+1186,3,2,CN,N/A
+1187,2,5,IT,N/A
+1188,3,2,JP,N/A
+1189,4,4,IT,N/A
+1190,4,3,GB,N/A
+1191,3,4,JP,N/A
+1192,2,2,FR,N/A
+1193,5,1,FR,N/A
+1194,5,2,US,N/A
+1195,2,5,FR,N/A
+1196,1,2,FR,N/A
+1197,5,3,FR,N/A
+1198,1,4,GB,N/A
+1199,1,3,FR,N/A
+1200,4,5,IT,N/A
+1201,3,1,GB,N/A
+1202,0,3,GB,N/A
+1203,4,3,DE,N/A
+1204,4,2,GB,N/A
+1205,4,2,DE,N/A
+1206,5,2,GB,N/A
+1207,1,5,DE,N/A
+1208,4,5,GB,N/A
+1209,0,3,FR,N/A
+1210,1,5,FR,N/A
+1211,3,2,DE,N/A
+1212,5,1,JP,N/A
+1213,0,4,CN,N/A
+1214,3,1,JP,N/A
+1215,5,1,CN,N/A
+1216,4,2,FR,N/A

[04/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index eb17922..48eef46 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -6,27 +6,32 @@
   "description" : null,
   "null_string" : null,
   "dimensions" : [ {
+"name" : "TRANS_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "TRANS_ID",
+"derived" : null
+  }, {
 "name" : "CAL_DT",
 "table" : "DEFAULT.KYLIN_CAL_DT",
 "column" : "{FK}",
-"derived" : [ "WEEK_BEG_DT" ]
+"derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
   }, {
 "name" : "CATEGORY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "{FK}",
 "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
   }, {
-"name" : "CATEGORY_HIERARCHY",
+"name" : "META_CATEG_NAME",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "META_CATEG_NAME",
 "derived" : null
   }, {
-"name" : "CATEGORY_HIERARCHY",
+"name" : "CATEG_LVL2_NAME",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "CATEG_LVL2_NAME",
 "derived" : null
   }, {
-"name" : "CATEGORY_HIERARCHY",
+"name" : "CATEG_LVL3_NAME",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "CATEG_LVL3_NAME",
 "derived" : null
@@ -36,14 +41,54 @@
 "column" : "LSTG_FORMAT_NAME",
 "derived" : null
   }, {
-"name" : "USER_ID",
+"name" : "SELLER_ID",
 "table" : "DEFAULT.KYLIN_SALES",
-"column" : "USER_ID",
+"column" : "SELLER_ID",
 "derived" : null
   }, {
-"name" : "REGION",
+"name" : "BUYER_ID",
 "table" : "DEFAULT.KYLIN_SALES",
-"column" : "REGION",
+"column" : "BUYER_ID",
+"derived" : null
+  }, {
+"name" : "ACCOUNT_BUYER_LEVEL",
+"table" : "BUYER_ACCOUNT",
+"column" : "ACCOUNT_BUYER_LEVEL",
+"derived" : null
+  }, {
+"name" : "ACCOUNT_SELLER_LEVEL",
+"table" : "SELLER_ACCOUNT",
+"column" : "ACCOUNT_SELLER_LEVEL",
+"derived" : null
+  }, {
+"name" : "BUYER_COUNTRY",
+"table" : "BUYER_ACCOUNT",
+"column" : "ACCOUNT_COUNTRY",
+"derived" : null
+  }, {
+"name" : "SELLER_COUNTRY",
+"table" : "SELLER_ACCOUNT",
+"column" : "ACCOUNT_COUNTRY",
+"derived" : null
+  }, {
+"name" : "BUYER_COUNTRY_NAME",
+"table" : "BUYER_COUNTRY",
+"column" : "NAME",
+"derived" : null
+  }, {
+"name" : "SELLER_COUNTRY_NAME",
+"table" : "SELLER_COUNTRY",
+"column" : "NAME",
+"derived" : null
+  }, {
+"name" : "OPS_USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "OPS_USER_ID",
+"derived" : null
+  }, {
+"name" : "OPS_REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "OPS_REGION",
 "derived" : null
   } ],
   "measures" : [ {
@@ -56,8 +101,7 @@
 "next_parameter" : null
   },
   "returntype" : "decimal(19,4)"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "GMV_MIN",
 "function" : {
@@ -68,8 +112,7 @@
 "next_parameter" : null
   },
   "returntype" : "decimal(19,4)"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "GMV_MAX",
 "function" : {
@@ -80,8 +123,7 @@
 "next_parameter" : null
   },
   "returntype" : "decimal(19,4)"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "TRANS_CNT",
 "function" : {
@@ -92,8 +134,7 @@
 "next_parameter" : null
   },
   "returntype" : "bigint"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "SELLER_CNT_HLL",
 "function" : {
@@ -104,20 +145,7 @@
 "next_parameter" : null
   },
   "returntype" : "hllc(10)"
-},
-"dependent_measure_ref" : null
-  }, {
-"name" : "SELLER_FORMAT_CNT",
-"function" : {
-  "expression" : "COUNT_DISTINCT",
-  "parameter" : {
-"type" : "column",
-"value" : "LSTG_FORMAT_NAME",
-"next_parameter" : null
-  },
-  "returntype" : "hllc(10)"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "TOP_SELLER",
 "function" : {
@@ -132,13 +160,21 @@
 }
   },
   "returntype" : "topn(100)"
-},
-"dependent_measure_ref" : null
+}
   } ],
   "rowkey" : {
 "rowkey_columns" : [ {
+  "column" : "BUYER_ID",
+  "encoding" : "integer:4"
+}, {
+  "column" : "SELLER_ID",
+  "encoding" : "integer:4"
+}, {
+  "column" : "TRANS_ID",
+  "encoding" : "integer:4"
+}, {
   "column" : "PART_DT",
-  "encoding" : "dict"
+  "encoding" : "date"
 }, {
   "column" : "LEAF_CATEG_ID",
   "encoding" :

[07/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'

2016-12-06 Thread lidong

KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 
'kylin_default_instance'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1a2a4a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1a2a4a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1a2a4a6

Branch: refs/heads/master-cdh5.7
Commit: a1a2a4a643d013471079ac0d353bbf179fd0d85c
Parents: c3d62e9
Author: Yang Li 
Authored: Sun Nov 27 18:26:33 2016 +0800
Committer: Yang Li 
Committed: Tue Nov 29 07:25:25 2016 +0800

--
 build/conf/kylin.properties | 2 +-
 .../apache/kylin/common/KylinConfigBase.java|11 +-
 .../kylin/metadata/model/DataModelDesc.java | 3 +
 .../kylin/metadata/model/MeasureDesc.java   | 5 +
 .../realization/IRealizationConstants.java  | 2 +-
 examples/sample_cube/create_sample_tables.sql   |21 +-
 .../sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv  | 1 +
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 .../template/cube_desc/kylin_sales_cube.json|   157 +-
 .../template/model_desc/kylin_sales_model.json  |50 +-
 .../template/table/DEFAULT.KYLIN_ACCOUNT.json   |28 +
 .../template/table/DEFAULT.KYLIN_SALES.json |12 +-
 .../test_case_data/sandbox/kylin.properties | 2 +-
 .../apache/kylin/query/relnode/OLAPJoinRel.java |25 +-
 .../kylin/rest/security/AclHBaseStorage.java| 1 -
 .../rest/security/RealAclHBaseStorage.java  | 5 +-
 .../apache/kylin/rest/service/QueryService.java | 6 +-
 .../kylin/storage/hbase/HBaseResourceStore.java | 2 +-
 .../storage/hbase/util/StorageCleanupJob.java   | 2 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 2 +-
 20 files changed, 20246 insertions(+), 10090 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 1409394..131a725 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -18,7 +18,7 @@
 ### METADATA | ENV ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
+kylin.metadata.url=kylin_default_instance@hbase
 
 # Working folder in HDFS, make sure user has the right access to the hdfs 
directory
 kylin.env.hdfs-working-dir=/kylin

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index aa5e6e1..7dcc771 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -188,13 +188,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public String getMetadataUrlPrefix() {
-String hbaseMetadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_metadata";
+String metadataUrl = getMetadataUrl();
+String defaultPrefix = "kylin_default_instance";
 
-if 
(org.apache.commons.lang3.StringUtils.containsIgnoreCase(hbaseMetadataUrl, 
"@hbase")) {
-int cut = hbaseMetadataUrl.indexOf('@');
-String tmp = cut < 0 ? defaultPrefix : 
hbaseMetadataUrl.substring(0, cut);
-return tmp;
+if (metadataUrl.endsWith("@hbase")) {
+int cut = metadataUrl.lastIndexOf('@');
+return metadataUrl.substring(0, cut);
 } else {
 return defaultPrefix;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index c2de5d6..ff92def 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -378,6 +378,9 @@ public class DataModelDesc extends RootPersistentEntity {
 for (int i = 0; i < pks.length; i++) {
 TblColRef col = dimTable.getColumn(pks[i]);
 if (col == null) {
+col = findColumn(pks[i]);
+}
+if (col == null || col.getTableRef().equals(dimTable) == 
false) {

[13/50] [abbrv] kylin git commit: KYLIN-2217 Reducers build dictionaries locally

2016-12-06 Thread lidong

KYLIN-2217 Reducers build dictionaries locally

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1af08e4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1af08e4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1af08e4b

Branch: refs/heads/master-cdh5.7
Commit: 1af08e4b8875d33bfc5dd124fed72d6042456c32
Parents: b1b90ad
Author: xiefan46 <958034...@qq.com>
Authored: Wed Nov 23 09:48:55 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 30 15:31:58 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   9 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   3 +-
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  |  25 ++-
 .../apache/kylin/dict/DictionaryManager.java|  17 +-
 .../apache/kylin/dict/DictionaryProvider.java   |  28 
 .../dict/DictionaryReducerLocalGenerator.java   | 156 +++
 .../dict/IDictionaryReducerLocalBuilder.java|  31 
 .../kylin/dict/DictionaryProviderTest.java  | 109 +
 .../storage/translate/ColumnValueRange.java |   2 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  44 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 123 +--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  54 +++
 .../storage/hbase/cube/v1/CubeStorageQuery.java |   6 +-
 13 files changed, 547 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7dcc771..766c04d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -719,7 +719,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 //UHC: ultra high cardinality columns, contain the ShardByColumns and the 
GlobalDictionaryColumns
 public int getUHCReducerCount() {
-return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
+return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
+}
+
+public boolean isReducerLocalBuildDict() {
+if (getUHCReducerCount() != 1) {
+return false;
+}
+return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
 }
 
 public String getYarnStatusCheckUrl() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b4422d2..119a21a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -213,6 +213,7 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
+
 public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
@@ -221,6 +222,7 @@ public class CubeManager implements IRealizationProvider {
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
 DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+
 saveDictionaryInfo(cubeSeg, col, dictInfo);
 return dictInfo;
 }
@@ -266,7 +268,6 @@ public class CubeManager implements IRealizationProvider {
 } catch (IOException e) {
 throw new IllegalStateException("Failed to get dictionary for cube 
segment" + cubeSeg + ", col" + col, e);
 }
-
 return (Dictionary) info.getDictionaryObject();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index a6aeb96..a4e1df0 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++

[18/50] [abbrv] kylin git commit: measure encoding length bug

2016-12-06 Thread lidong

measure encoding length bug

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a5295ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a5295ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a5295ba

Branch: refs/heads/master-cdh5.7
Commit: 1a5295bad21fc18766bb3c7ba6eee59678d42185
Parents: 5822492
Author: luguosheng <550175...@qq.com>
Authored: Thu Dec 1 17:54:35 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 18:17:35 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 4 ++--
 webapp/app/js/controllers/cubeEdit.js   | 2 +-
 webapp/app/js/controllers/cubeMeasures.js   | 8 
 3 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index f2b36fe..8192419 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -47,7 +47,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var _encoding = item.encoding;
 var _valueLength ;
 var baseKey=item.encoding.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>=-1){
+if(needLengthKeyList.indexOf(baseKey)!=-1){
   var result=/:(\d+)/.exec(item.encoding);
   _valueLength=result?result[1]:0;
 }
@@ -78,7 +78,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var version=$scope.getTypeVersion(item.encoding);
 var encodingType=$scope.removeVersion(item.encoding);
 
-if(needLengthKeyList.indexOf(encodingType)>=-1){
+if(needLengthKeyList.indexOf(encodingType)!=-1){
   encoding = encodingType+":"+item.valueLength;
 }else{
   encoding = encodingType;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 0704a62..85bd4b1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -78,7 +78,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 $scope.store.supportedEncoding = $scope.cubeConfig.encodings;
   })
   $scope.createFilter=function(type){
- if(type.indexOf("varchar")<=0){
+ if(type.indexOf("varchar")==-1){
return ['fixed_length_hex'];
  }else if(type!="date"){
return ['date'];

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 085338f..18f53c2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -45,7 +45,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 }
   }
   $scope.createFilter=function(type){
-if(type.indexOf("varchar")<=0){
+if(type.indexOf("varchar")==-1){
   return ['fixed_length_hex'];
 }else if(type!="date"){
   return ['date'];
@@ -60,7 +60,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var encodings =$scope.store.supportedEncoding,filterEncoding=[];
 var filerList=$scope.createFilter(type);
 if($scope.isEdit) {
-  if (name && $scope.newMeasure.function.configuration) {
+  if (name && 
$scope.newMeasure.function.configuration&&$scope.newMeasure.function.configuration['topn.encoding.'
 + name]) {
 var version = 
$scope.newMeasure.function.configuration['topn.encoding_version.' + name] || 1;
 filterEncoding = VdmUtil.getFilterObjectListByOrFilterVal(encodings, 
'value', $scope.newMeasure.function.configuration['topn.encoding.' + 
name].replace(/:\d+/, "") + (version ? "[v" + version + "]" : "[v1]"), 
'suggest', true);
   }else{
@@ -119,7 +119,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var 
version=$scope.newMeasure.function.configuration['topn.encoding_version.'+_name]||1;
 item=$scope.removeVersion(item);
 var baseKey=item.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>=-1){
+if(needLengthKeyList.indexOf(baseKey)!=-1){
   var result=/:(\d+)/.exec(item);

[12/50] [abbrv] kylin git commit: minor, massin udf reverse

2016-12-06 Thread lidong

minor, massin udf reverse

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1b90adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1b90adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1b90adb

Branch: refs/heads/master-cdh5.7
Commit: b1b90adbe896e808776caf49ad7172979cf860df
Parents: 875a593
Author: Roger Shi 
Authored: Wed Nov 30 13:35:12 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 30 13:49:16 2016 +0800

--
 .../metadata/filter/UDF/MassInTupleFilter.java  | 28 +++-
 1 file changed, 27 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b1b90adb/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 3f0546c..29c5550 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -50,6 +50,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 private String filterTableName;//key in MetadataManager.extFilterMap
 private String filterTableResourceIdentifier;//HDFS path, or hbase table 
name depending on FilterTableType
 private Functions.FilterTableType filterTableType;
+private boolean reverse = false;
 
 public MassInTupleFilter() {
 super(Lists. newArrayList(), 
TupleFilter.FilterOperatorEnum.MASSIN);
@@ -66,7 +67,18 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 valueProvider = 
VALUE_PROVIDER_FACTORY.getProvider(filterTableType, 
filterTableResourceIdentifier, column);
 }
 boolean ret = valueProvider.getMassInValues().contains(colValue);
-return ret;
+return reverse ? !ret : ret;
+}
+
+@Override
+public TupleFilter reverse() {
+try {
+MassInTupleFilter result = (MassInTupleFilter) this.clone();
+result.setReverse(!this.isReverse());
+return result;
+} catch (CloneNotSupportedException e) {
+throw new UnsupportedOperationException(e);
+}
 }
 
 @Override
@@ -144,4 +156,18 @@ public class MassInTupleFilter extends FunctionTupleFilter 
{
 return false;
 }
 
+public boolean isReverse() {
+return reverse;
+}
+
+public void setReverse(boolean reverse) {
+this.reverse = reverse;
+}
+
+@Override
+protected Object clone() throws CloneNotSupportedException {
+MassInTupleFilter result = new MassInTupleFilter();
+result.setReverse(this.isReverse());
+return result;
+}
 }

[39/50] [abbrv] kylin git commit: KYLIN-2245 further minor refactor

2016-12-06 Thread lidong

KYLIN-2245 further minor refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9968c225
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9968c225
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9968c225

Branch: refs/heads/master-cdh5.7
Commit: 9968c2257c2a8605637a93830d1d95db94185dce
Parents: fc1e11a
Author: Li Yang 
Authored: Tue Dec 6 11:10:26 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 6 11:10:26 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |  2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |  3 ++-
 .../apache/kylin/metadata/model/ISegment.java   |  3 +++
 .../apache/kylin/metadata/model/Segments.java   | 15 +++
 .../test_case_data/sandbox/kylin_hive_conf.xml  | 20 +---
 5 files changed, 10 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a3665f5..1d60575 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -368,7 +368,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 public Segments calculateToBeSegments(CubeSegment newSegment) {
-return segments.calculateToBeSegments(newSegment, 
getModel().getPartitionDesc().isPartitioned());
+return segments.calculateToBeSegments(newSegment);
 }
 
 public CubeSegment getLastSegment() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 5339d81..e155f86 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -361,7 +361,8 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 return Segments.sourceOffsetContains(this, seg);
 }
 
-public void validate() {
+@Override
+public void validate() throws IllegalStateException {
 if 
(cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
 if (!isSourceOffsetsOn() && dateRangeStart >= dateRangeEnd)
 throw new IllegalStateException("Invalid segment, 
dateRangeStart(" + dateRangeStart + ") must be smaller than dateRangeEnd(" + 
dateRangeEnd + ") in segment " + this);

http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index f006613..d46ea96 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -37,4 +37,7 @@ public interface ISegment {
 public SegmentStatusEnum getStatus();
 
 public long getLastBuildTime();
+
+public void validate() throws IllegalStateException;
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 5198dc7..9371f76 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -229,7 +229,7 @@ public class Segments extends 
ArrayList {
  * - Favors new segments over the old
  * - Favors big segments over the small
  */
-public Segments calculateToBeSegments(ISegment newSegment, boolean 
isPartitioned) {
+public Segments calculateToBeSegments(ISegment newSegment) {
 
 Segments tobe = (Segments) this.clone();
 if (newSegment != null && !tobe.contains(newSegment)) {
@@ -242,12 +242,12 @@ public class Segments extends 
ArrayList {
 Collections.sort(tobe);
 
 ISegment firstSeg = tobe.getFirst();
-

[21/50] [abbrv] kylin git commit: KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally

2016-12-06 Thread lidong

KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e562aafe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e562aafe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e562aafe

Branch: refs/heads/master-cdh5.7
Commit: e562aafee10c8bbd125937503d42b958c1843aad
Parents: 3091f06
Author: Hongbin Ma 
Authored: Fri Dec 2 13:33:02 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 2 13:33:18 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java   | 18 ++
 .../java/org/apache/kylin/jdbc/IRemoteClient.java |  3 ++-
 .../java/org/apache/kylin/jdbc/KylinClient.java   |  7 ---
 .../org/apache/kylin/jdbc/KylinResultSet.java |  9 -
 .../org/apache/kylin/jdbc/json/QueryRequest.java  | 12 
 .../java/org/apache/kylin/jdbc/DummyClient.java   |  3 ++-
 .../apache/kylin/rest/service/QueryService.java   | 11 ++-
 7 files changed, 56 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index f7c90aa..28f7697 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -79,6 +79,14 @@ public class BackdoorToggles {
 }
 }
 
+public static Integer getStatementMaxRows() {
+String v = getString(ATTR_STATEMENT_MAX_ROWS);
+if (v == null)
+return null;
+else
+return Integer.valueOf(v);
+}
+
 private static String getString(String key) {
 Map toggles = _backdoorToggles.get();
 if (toggles == null) {
@@ -183,4 +191,14 @@ public class BackdoorToggles {
  */
 public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = 
"DEBUG_TOGGLE_SHARD_ASSIGNMENT";
 
+// properties on statement may go with this "channel" too
+/**
+ * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "ATTR_STATEMENT_MAX_ROWS": "10"
+ }
+ */
+public final static String ATTR_STATEMENT_MAX_ROWS = 
"ATTR_STATEMENT_MAX_ROWS";
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
index b6a13e5..dfd8d76 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.calcite.avatica.AvaticaParameter;
 import org.apache.calcite.avatica.ColumnMetaData;
@@ -51,6 +52,6 @@ public interface IRemoteClient extends Closeable {
 /**
  * Execute query remotely and get back result.
  */
-public QueryResult executeQuery(String sql, List params, 
List paramValues) throws IOException;
+public QueryResult executeQuery(String sql, List params, 
List paramValues, Map queryToggles) throws IOException;
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index 2d06a92..86c3a5b 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -321,9 +321,9 @@ public class KylinClient implements IRemoteClient {
 }
 
 @Override
-public QueryResult executeQuery(String sql, List params, 
List paramValues) throws IOException {
+public QueryResult executeQuery(String sql, List params, 
List paramValues, Map queryToggles) throws IOException {
 
-SQLResponseStub queryResp = executeKylinQuery(sql, 
convertParameters(params, paramValues));
+SQLResponseStub queryResp = executeKylinQuery(sql, 
convertParameters(params, paramValues), queryToggles);
 if

[27/50] [abbrv] kylin git commit: KYLIN-2245 slim ISegment a bit

2016-12-06 Thread lidong

KYLIN-2245 slim ISegment a bit


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0eebd7d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0eebd7d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0eebd7d8

Branch: refs/heads/master-cdh5.7
Commit: 0eebd7d8ab10b6b2b4823d6a23eaa774ac36989e
Parents: 3ca5395
Author: Li Yang 
Authored: Fri Dec 2 17:23:16 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 17:26:13 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeSegment.java | 20 +++-
 .../apache/kylin/metadata/model/ISegment.java   |  1 -
 .../apache/kylin/metadata/model/Segments.java   | 12 +++-
 3 files changed, 14 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 1fc28eb..5339d81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -38,6 +38,7 @@ import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.ISegment;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.IRealization;
 
@@ -350,29 +351,14 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 this.sourceOffsetEnd = sourceOffsetEnd;
 }
 
-public boolean dateRangeOverlaps(CubeSegment seg) {
-return dateRangeStart < seg.dateRangeEnd && seg.dateRangeStart < 
dateRangeEnd;
-}
-
-public boolean dateRangeContains(CubeSegment seg) {
-return dateRangeStart <= seg.dateRangeStart && seg.dateRangeEnd <= 
dateRangeEnd;
-}
-
 // date range is used in place of source offsets when offsets are missing
 public boolean sourceOffsetOverlaps(CubeSegment seg) {
-if (isSourceOffsetsOn())
-return sourceOffsetStart < seg.sourceOffsetEnd && 
seg.sourceOffsetStart < sourceOffsetEnd;
-else
-return dateRangeOverlaps(seg);
+return Segments.sourceOffsetOverlaps(this, seg);
 }
 
 // date range is used in place of source offsets when offsets are missing
-@Override
 public boolean sourceOffsetContains(ISegment seg) {
-if (isSourceOffsetsOn())
-return sourceOffsetStart <= ((CubeSegment) seg).sourceOffsetStart 
&& ((CubeSegment) seg).sourceOffsetEnd <= sourceOffsetEnd;
-else
-return dateRangeContains(((CubeSegment) seg));
+return Segments.sourceOffsetContains(this, seg);
 }
 
 public void validate() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index e97f4f4..e3fcdcb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -36,5 +36,4 @@ public interface ISegment{
 
 public long getLastBuildTime();
 
-public boolean sourceOffsetContains(ISegment seg);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 104c2af..f0a58cb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -21,6 +21,16 @@ package org.apache.kylin.metadata.model;
 import java.util.ArrayList;
 
 public class Segments extends ArrayList {
+
+private static final long serialVersionUID = 1L;
+
+public static boolean sourceOffsetContains(ISegment a, ISegment b) {
+return a.getSourceOffsetStart() <= b.getSourceOffsetStart() && 
b.getSourceOffsetEnd() <= a.getSourceOffsetEnd();
+}
+
+public static boolean sourceOffsetOverlaps(ISegment a, ISegment b) {
+return a.getSourceOffsetStart() <

[14/50] [abbrv] kylin git commit: KYLIN-2217 Code review, refactor IDictionaryBuilder

2016-12-06 Thread lidong

KYLIN-2217 Code review, refactor IDictionaryBuilder


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30cb1ac6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30cb1ac6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30cb1ac6

Branch: refs/heads/master-cdh5.7
Commit: 30cb1ac678b063ac164acd27591e8a1d0becafa0
Parents: 1af08e4
Author: Li Yang 
Authored: Wed Nov 30 15:30:15 2016 +0800
Committer: Yang Li 
Committed: Wed Nov 30 21:00:17 2016 +0800

--
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  |   2 +-
 .../apache/kylin/dict/DictionaryGenerator.java  | 165 ---
 .../apache/kylin/dict/DictionaryManager.java|  16 +-
 .../apache/kylin/dict/DictionaryProvider.java   |   4 +-
 .../dict/DictionaryReducerLocalGenerator.java   | 156 --
 .../kylin/dict/GlobalDictionaryBuilder.java |  36 ++--
 .../apache/kylin/dict/IDictionaryBuilder.java   |  13 +-
 .../dict/IDictionaryReducerLocalBuilder.java|  31 
 .../kylin/dict/DictionaryProviderTest.java  |  63 +++
 .../engine/mr/steps/CreateDictionaryJob.java|  42 ++---
 .../mr/steps/FactDistinctColumnsReducer.java| 115 +
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   9 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   6 +-
 13 files changed, 231 insertions(+), 427 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index a4e1df0..163c6ca 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -60,7 +60,7 @@ public class DictionaryGeneratorCLI {
 for (TblColRef col : 
cubeSeg.getCubeDesc().getAllColumnsNeedDictionaryBuilt()) {
 logger.info("Building dictionary for " + col);
 ReadableTable inpTable = decideInputTable(cubeSeg.getModel(), col, 
factTableValueProvider);
-if (config.isReducerLocalBuildDict() && dictProvider != null) {
+if (dictProvider != null) {
 Dictionary dict = dictProvider.getDictionary(col);
 if (dict != null) {
 cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);

http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 810a392..cd13d59 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -19,12 +19,11 @@
 package org.apache.kylin.dict;
 
 import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.slf4j.Logger;
@@ -40,9 +39,7 @@ public class DictionaryGenerator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
-private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
-
-public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
+public static IDictionaryBuilder newDictionaryBuilder(DataType dataType) {
 Preconditions.checkNotNull(dataType, "dataType cannot be null");
 
 // build dict, case by data type
@@ -57,16 +54,33 @@ public class DictionaryGenerator {
 } else {
 builder = new StringDictBuilder();
 }
+return builder;
+}
 
-return buildDictionary(builder, null, valueEnumerator);
+public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
+return buildDictionary(newDictionaryBuilder(dataType), null, 
valueEnumerator);
 }
 
-public static Dictionary buildDictionary(IDictionaryBuilder 
builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) 
throws IOException {
+static

[35/50] [abbrv] kylin git commit: KYLIN-2212 add more test queries

2016-12-06 Thread lidong

KYLIN-2212 add more test queries


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af429e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af429e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af429e5c

Branch: refs/heads/master-cdh5.7
Commit: af429e5cbbda2a64159b25bc5ce4ad70f2f1a1f2
Parents: 8f3239b
Author: Hongbin Ma 
Authored: Mon Dec 5 15:05:35 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 5 15:05:35 2016 +0800

--
 .../metadata/filter/LogicalTupleFilter.java |  3 +-
 .../src/test/resources/query/sql/query101.sql   | 13 
 .../src/test/resources/query/sql/query102.sql   | 13 
 .../src/test/resources/query/sql/query103.sql   | 13 
 .../test/resources/query/sql_like/query21.sql   | 31 
 5 files changed, 71 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 61657fb..1744309 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -64,8 +64,7 @@ public class LogicalTupleFilter extends TupleFilter {
 public TupleFilter reverse() {
 switch (operator) {
 case NOT:
-throw new IllegalStateException("not( not in ()) is invalid 
syntax");
-//return reverseNestedNots(this, 0);
+throw new IllegalStateException("NOT will be replaced in 
org.apache.kylin.query.relnode.OLAPFilterRel.TupleFilterVisitor");
 case AND:
 case OR:
 LogicalTupleFilter reverse = new 
LogicalTupleFilter(REVERSE_OP_MAP.get(operator));

http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query101.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query101.sql 
b/kylin-it/src/test/resources/query/sql/query101.sql
new file mode 100644
index 000..fb42bca
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query101.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV 
+
+ from test_kylin_fact 
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') )
+ group by meta_categ_name 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query102.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query102.sql 
b/kylin-it/src/test/resources/query/sql/query102.sql
new file mode 100644
index 000..bd1e15e
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query102.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV 
+
+ from test_kylin_fact 
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') and meta_categ_name 
not in ('Crafts','Computers'))
+ group by meta_categ_name 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query103.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query103.sql 
b/kylin-it/src/test/resources/query/sql/query103.sql
new file mode 100644
index 000..c5f9bf9
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query103.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV 
+
+ from test_kylin_fact 
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON

[32/50] [abbrv] kylin git commit: minor, change isReverse visibility in BuiltInFunctionTupleFilter.java

2016-12-06 Thread lidong

minor, change isReverse visibility in BuiltInFunctionTupleFilter.java


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00a4c506
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00a4c506
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00a4c506

Branch: refs/heads/master-cdh5.7
Commit: 00a4c5064bfe894bb8c106a214277d4d1d0f3c70
Parents: a03eec8
Author: Hongbin Ma 
Authored: Sun Dec 4 21:41:07 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Dec 4 21:41:07 2016 +0800

--
 .../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/00a4c506/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index f6e687b..b678394 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -47,7 +47,7 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 protected Method method;
 protected List methodParams;
 protected boolean isValidFunc = false;
-private boolean isReversed = false;
+protected boolean isReversed = false;
 
 public BuiltInFunctionTupleFilter(String name) {
 this(name, null);

[31/50] [abbrv] kylin git commit: minor, refine MassInTupleFilter

2016-12-06 Thread lidong

minor, refine MassInTupleFilter

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a03eec88
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a03eec88
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a03eec88

Branch: refs/heads/master-cdh5.7
Commit: a03eec88f3b4e60d1f1d6bd2fd5922a97aec23f5
Parents: fe9efa8
Author: Roger Shi 
Authored: Sun Dec 4 16:28:47 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Dec 4 21:28:17 2016 +0800

--
 .../metadata/filter/UDF/MassInTupleFilter.java  | 34 +---
 1 file changed, 30 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a03eec88/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 29c5550..e4e311e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.metadata.filter.UDF;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collection;
 
 import org.apache.kylin.common.KylinConfig;
@@ -40,7 +41,6 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
 public class MassInTupleFilter extends FunctionTupleFilter {
-
 public static final Logger logger = 
LoggerFactory.getLogger(MassInTupleFilter.class);
 public static MassInValueProviderFactory VALUE_PROVIDER_FACTORY = null;
 
@@ -56,6 +56,16 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 super(Lists. newArrayList(), 
TupleFilter.FilterOperatorEnum.MASSIN);
 }
 
+public MassInTupleFilter(MassInTupleFilter filter) {
+super(new ArrayList(filter.children), filter.operator);
+this.valueProvider = filter.getValueProvider();
+this.column = filter.getColumn();
+this.filterTableName = filter.getFilterTableName();
+this.filterTableResourceIdentifier = 
filter.getFilterTableResourceIdentifier();
+this.filterTableType = filter.getFilterTableType();
+this.reverse = filter.isReverse();
+}
+
 @Override
 public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
 Preconditions.checkNotNull(tuple);
@@ -132,6 +142,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 BytesUtil.writeUTFString(filterTableName, buffer);
 BytesUtil.writeUTFString(filterTableResourceIdentifier, buffer);
 BytesUtil.writeUTFString(filterTableType.toString(), buffer);
+BytesUtil.writeUTFString(String.valueOf(reverse), buffer);
 }
 
 @Override
@@ -139,6 +150,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 filterTableName = BytesUtil.readUTFString(buffer);
 filterTableResourceIdentifier = BytesUtil.readUTFString(buffer);
 filterTableType = 
Functions.FilterTableType.valueOf(BytesUtil.readUTFString(buffer));
+reverse = Boolean.valueOf(BytesUtil.readUTFString(buffer));
 }
 
 public static boolean containsMassInTupleFilter(TupleFilter filter) {
@@ -166,8 +178,22 @@ public class MassInTupleFilter extends FunctionTupleFilter 
{
 
 @Override
 protected Object clone() throws CloneNotSupportedException {
-MassInTupleFilter result = new MassInTupleFilter();
-result.setReverse(this.isReverse());
-return result;
+return new MassInTupleFilter(this);
+}
+
+public MassInValueProvider getValueProvider() {
+return valueProvider;
+}
+
+public String getFilterTableName() {
+return filterTableName;
+}
+
+public String getFilterTableResourceIdentifier() {
+return filterTableResourceIdentifier;
+}
+
+public Functions.FilterTableType getFilterTableType() {
+return filterTableType;
 }
 }

[47/50] [abbrv] kylin git commit: minor, chmod +x on scripts

2016-12-06 Thread lidong

minor, chmod +x on scripts


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/081ed0d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/081ed0d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/081ed0d5

Branch: refs/heads/master-cdh5.7
Commit: 081ed0d505e5638b1294c59e57ed651a0f9dffa1
Parents: 1e78716
Author: lidongsjtu 
Authored: Wed Dec 7 11:10:49 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 11:10:49 2016 +0800

--
 dev-support/sync_hbase_cdh_branches.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/081ed0d5/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
old mode 100644
new mode 100755

[10/50] [abbrv] kylin git commit: Revert "KYLIN-2195 could run get-properties.sh in source code dir"

2016-12-06 Thread lidong

Revert "KYLIN-2195 could run get-properties.sh in source code dir"

This reverts commit 8506e4d4f75fbbda518997cac97e229fa5a0.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3186d176
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3186d176
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3186d176

Branch: refs/heads/master-cdh5.7
Commit: 3186d176113596c71a47d98a927564cefff3b123
Parents: 8506e4d
Author: lidongsjtu 
Authored: Tue Nov 29 19:11:50 2016 +0800
Committer: lidongsjtu 
Committed: Tue Nov 29 19:11:50 2016 +0800

--
 build/bin/get-properties.sh | 12 ++--
 1 file changed, 2 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3186d176/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 0fa4f6f..170442d 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -19,21 +19,13 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
-
 if [ $# != 1 ]
 then
 echo 'invalid input'
 exit -1
 fi
 
-if [[ $CI_MODE == 'true' ]]; then
-cd $dir
-job_jar=$(ls ../../assembly/target/kylin-*-job.jar)
-tool_jar=$(ls ../../tool/target/kylin-tool-*.jar|grep -v assembly)
-else
-job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
-tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-fi
-
+job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
 result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 
2>/dev/null`
 echo "$result"

[28/50] [abbrv] kylin git commit: KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result

2016-12-06 Thread lidong

KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fd1ed6f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fd1ed6f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fd1ed6f

Branch: refs/heads/master-cdh5.7
Commit: 0fd1ed6f14758bb38f1abf7a7ed9e24ed818f50f
Parents: 0eebd7d
Author: zhengdong 
Authored: Wed Nov 23 16:13:59 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 18:32:24 2016 +0800

--
 .../java/org/apache/kylin/gridtable/GTUtil.java | 14 
 .../metadata/filter/CompareTupleFilter.java |  2 ++
 .../metadata/filter/TupleFilterSerializer.java  |  1 +
 .../resources/query/sql_derived/query12.sql | 24 
 .../kylin/query/relnode/OLAPFilterRel.java  |  3 +++
 .../common/coprocessor/FilterDecorator.java | 14 
 6 files changed, 58 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
old mode 100644
new mode 100755
index a4e574c..7496778
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -196,6 +196,20 @@ public class GTUtil {
 result = newCompareFilter;
 }
 break;
+case NOTIN:
+Set notInValues = Sets.newHashSet();
+for (Object value : constValues) {
+code = translate(col, value, 0);
+if (code != null)
+notInValues.add(code);
+}
+if (notInValues.isEmpty()) {
+result = ConstantTupleFilter.TRUE;
+} else {
+newCompareFilter.addChild(new 
ConstantTupleFilter(notInValues));
+result = newCompareFilter;
+}
+break;
 case NEQ:
 code = translate(col, firstValue, 0);
 if (code == null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
old mode 100644
new mode 100755
index c7a3721..f2af735
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -59,6 +59,8 @@ public class CompareTupleFilter extends TupleFilter {
 private CompareTupleFilter(CompareTupleFilter another) {
 super(new ArrayList(another.children), another.operator);
 this.column = another.column;
+this.firstCondValue = another.getFirstValue();
+this.function = another.getFunction();
 this.conditionValues = new HashSet();
 this.conditionValues.addAll(another.conditionValues);
 this.dynamicVariables = new HashMap();

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
old mode 100644
new mode 100755
index 2df474e..63153ef
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -170,6 +170,7 @@ public class TupleFilterSerializer {
 case GT:
 case GTE:
 case IN:
+case NOTIN:
 case ISNULL:
 case ISNOTNULL:
 filter = new CompareTupleFilter(op);

http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/kylin-it/src/test/resources/query/sql_derived/query12.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql 
b/kylin-it/src/test/resources/query/sql_derived/query12.sql
new file mode 100755
index 000..959a59c
---

[15/50] [abbrv] kylin git commit: org.apache.kylin.metadata.model.FunctionDesc#configuration should use default linkedhashmap to keep order

2016-12-06 Thread lidong

org.apache.kylin.metadata.model.FunctionDesc#configuration should use default 
linkedhashmap to keep order


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/93bf0d02
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/93bf0d02
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/93bf0d02

Branch: refs/heads/master-cdh5.7
Commit: 93bf0d0214fe446d794b61e02fb18ff8097611d1
Parents: 30cb1ac
Author: Hongbin Ma 
Authored: Thu Dec 1 11:20:53 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 11:40:25 2016 +0800

--
 .../apache/kylin/common/util/JacksonBean.java| 16 
 .../apache/kylin/common/util/JacksonTest.java| 19 +++
 .../kylin/metadata/model/FunctionDesc.java   |  8 
 3 files changed, 35 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
index 42357f2..6d1 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -23,6 +23,10 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class JacksonBean {
 
@@ -32,6 +36,10 @@ public class JacksonBean {
 @JsonInclude(JsonInclude.Include.NON_NULL)
 private int b;
 
+@JsonProperty("configuration")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+private Map configuration = new HashMap();
+
 public String getA() {
 return a;
 }
@@ -48,6 +56,14 @@ public class JacksonBean {
 this.b = b;
 }
 
+public Map getConfiguration() {
+return configuration;
+}
+
+public void setConfiguration(Map configuration) {
+this.configuration = configuration;
+}
+
 @Override
 public String toString() {
 return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';

http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
index 81be7eb..0d7097a 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -18,22 +18,33 @@
 
 package org.apache.kylin.common.util;
 
-import java.io.IOException;
-
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 public class JacksonTest {
 @Test
 public void foo() throws IOException {
+HashMap a = new HashMap();
+a.put("1", "1");
+a.put("3", "3");
+a.put("2", "2");
+
+
 JacksonBean bean = new JacksonBean();
 bean.setA("valuea");
+bean.setConfiguration(a);
 
 String s = JsonUtil.writeValueAsString(bean);
 System.out.println(s);
 
-JacksonBean desBean = (JacksonBean) 
JsonUtil.readValue("{\"a\":\"valuea\"}", JacksonBean.class);
+JacksonBean desBean = (JacksonBean) 
JsonUtil.readValue("{\"a\":\"valuea\",\"b\":0,\"configuration\":{\"2\":\"2\",\"3\":\"3\",\"1\":\"1\"}}",
 JacksonBean.class);
+
 String x2 = JsonUtil.writeValueAsString(desBean);
-System.out.println(desBean);
 System.out.println(x2);
+
+System.out.println(desBean);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index ae7f805..7b41552 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++

[40/50] [abbrv] kylin git commit: minor, generate item_count value for sample data

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/d2aaf270/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index 7349b37..e13b7ad 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,1349,10002313,ANALYST,Beijing
-1,2012-08-28,Others,175750,0,13,23.8563,0,1927,10004376,ANALYST,Beijing
-2,2012-02-16,ABIN,148324,15,13,88.3418,0,1005,10006710,ADMIN,Shanghai
-3,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,1209,10003717,ANALYST,Beijing
-4,2012-10-22,Others,140746,100,11,83.454,0,1154,10006076,ADMIN,Shanghai
-5,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,1372,10007436,ADMIN,Shanghai
-6,2013-04-04,Others,963,0,13,88.5907,0,1648,10009869,MODELER,Hongkong
-7,2012-04-11,Others,15687,0,15,88.194,0,1866,1400,ADMIN,Shanghai
-8,2013-01-19,ABIN,60606,3,13,77.9727,0,1936,1687,MODELER,Hongkong
-9,2012-04-30,FP-non GTC,106246,0,14,52.295,0,1047,10009223,MODELER,Hongkong
-10,2013-02-03,Auction,45333,0,16,56.3584,0,1470,10006759,MODELER,Hongkong
-11,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,1203,1062,ANALYST,Beijing
-12,2013-07-30,Others,24760,0,16,25.077,0,1382,10003539,ADMIN,Shanghai
-13,2012-04-17,Auction,31519,0,16,72.7384,0,1069,10001245,ADMIN,Shanghai
-14,2013-04-30,FP-GTC,51582,0,15,75.82,0,1085,10008148,ANALYST,Beijing
-15,2013-03-23,FP-GTC,16509,0,15,8.6653,0,1559,10008079,MODELER,Hongkong
-16,2013-04-01,ABIN,175750,3,16,64.2802,0,1507,10007797,MODELER,Hongkong
-17,2013-07-12,ABIN,50677,0,-99,24.5987,0,1938,10006224,ADMIN,Shanghai
-18,2012-04-23,FP-GTC,45333,0,5,72.6553,0,1863,10003824,MODELER,Hongkong
-19,2012-12-19,ABIN,3838,0,12,87.4142,0,1612,10007157,ADMIN,Shanghai
-20,2012-08-15,FP-GTC,62179,0,16,67.4238,0,1839,10003452,MODELER,Hongkong
-21,2013-08-22,Auction,1357,0,5,18.36,0,1303,1114,ANALYST,Beijing
-22,2013-08-23,ABIN,31519,0,5,67.6344,0,1877,10007417,ANALYST,Beijing
-23,2013-09-12,FP-GTC,223,0,14,71.7898,0,1848,1627,MODELER,Hongkong
-24,2012-03-14,FP-GTC,132939,0,15,26.1496,0,1815,10005292,MODELER,Hongkong
-25,2013-07-13,Others,67703,3,16,83.9661,0,1858,10008123,MODELER,Hongkong
-26,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,1387,10007806,ADMIN,Shanghai
-27,2012-08-30,Auction,62179,0,11,21.0496,0,1393,10007360,ANALYST,Beijing
-28,2013-01-10,FP-GTC,48027,0,13,23.4142,0,1757,10002135,MODELER,Hongkong
-29,2013-01-25,ABIN,145970,0,5,26.7842,0,1402,10009682,ANALYST,Beijing
-30,2012-07-05,Auction,48027,0,-99,46.732,0,1972,10009522,MODELER,Hongkong
-31,2012-11-07,FP-GTC,20213,0,-99,19.727,0,1873,10009944,ANALYST,Beijing
-32,2013-06-20,ABIN,100847,0,5,76.2783,0,1409,10005395,ADMIN,Shanghai
-33,2012-10-08,ABIN,26262,0,5,48.7447,0,1029,10001836,ADMIN,Shanghai
-34,2013-04-30,ABIN,87118,0,15,63.2187,0,1529,1928,ANALYST,Beijing
-35,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,1329,1768,ANALYST,Beijing
-36,2012-05-13,Others,164261,0,13,84.1246,0,1688,1727,ADMIN,Shanghai
-37,2013-08-02,ABIN,31519,0,5,37.1504,0,1905,1719,MODELER,Hongkong
-38,2012-04-03,Auction,164261,0,11,92.0974,0,1341,10001337,ADMIN,Shanghai
-39,2012-09-02,FP-GTC,31519,0,14,87.7829,0,1460,10008484,ANALYST,Beijing
-40,2013-10-05,ABIN,13987,0,16,74.0719,0,1381,10002975,MODELER,Hongkong
-41,2012-01-13,FP-GTC,48027,0,15,71.2637,0,1915,10005778,ADMIN,Shanghai
-42,2013-03-01,Others,13836,0,14,16.7288,0,1953,10009385,ADMIN,Shanghai
-43,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,1277,10002286,MODELER,Hongkong
-44,2012-02-07,Auction,45333,0,5,64.977,0,1040,10005583,MODELER,Hongkong
-45,2013-12-14,FP-non 
GTC,158798,0,16,72.4413,0,1500,10007635,MODELER,Hongkong
-46,2013-10-13,Auction,31519,0,5,79.3053,0,1816,10006423,MODELER,Hongkong
-47,2012-06-11,ABIN,4943,0,5,11.6942,0,1696,10001237,ANALYST,Beijing
-48,2012-10-18,ABIN,80053,0,-99,54.0933,0,1604,10006851,MODELER,Hongkong
-49,2012-11-03,Auction,20886,0,5,9.8258,0,1764,10001220,MODELER,Hongkong
-50,2012-01-15,Auction,44079,0,14,13.0371,0,1343,10001259,MODELER,Hongkong
-51,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,1724,10009008,ADMIN,Shanghai
-52,2012-02-16,FP-GTC,314,0,13,49.1825,0,1334,10004005,ADMIN,Shanghai
-53,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,1603,10002595,ANALYST,Beijing
-54,2012-07-15,FP-non 
GTC,32876,0,-99,50.9634,0,1248,10006890,ANALYST,Beijing
-55,2012-07-24,FP-GTC,150265,15,16,57.5645,0,1653,10007269,ADMIN,Shanghai
-56,2012-10-06,Auction,145970,0,5,44.3091,0,1082,10001640,ADMIN,Shanghai
-57,2013-06-14,ABIN,80053,0,5,20.2388,0,1632,10002068,MODELER,Hongkong
-58,2013-10-10,FP-non

[43/50] [abbrv] kylin git commit: KYLIN-2250 HiveMRInput accepts intermediate_view_table_name

2016-12-06 Thread lidong

KYLIN-2250 HiveMRInput accepts intermediate_view_table_name

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5c05880
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5c05880
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5c05880

Branch: refs/heads/master-cdh5.7
Commit: b5c058801b916ddc6e00085d652ad6a0497ce7d9
Parents: 7699973
Author: Cheng Wang 
Authored: Tue Dec 6 15:48:21 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 6 16:46:40 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeManager.java  |  2 +-
 .../apache/kylin/cube/cli/DictionaryGeneratorCLI.java |  2 +-
 .../org/apache/kylin/metadata/model/TableDesc.java| 14 +-
 .../org/apache/kylin/source/hive/HiveMRInput.java | 13 +
 4 files changed, 20 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 0c80c07..bda1423 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -274,7 +274,7 @@ public class CubeManager implements IRealizationProvider {
 SnapshotManager snapshotMgr = getSnapshotManager();
 
 TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable));
-if 
(TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+if (tableDesc.isView()) {
 String tableName = tableDesc.getMaterializedName();
 
tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable());
 tableDesc.setName(tableName);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 163c6ca..3e1ab0d 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -98,7 +98,7 @@ public class DictionaryGeneratorCLI {
 } else {
 MetadataManager metadataManager = 
MetadataManager.getInstance(config);
 TableDesc tableDesc = new 
TableDesc(metadataManager.getTableDesc(srcTable));
-if 
(TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+if (tableDesc.isView()) {
 TableDesc materializedTbl = new TableDesc();
 
materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
 materializedTbl.setName(tableDesc.getMaterializedName());

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 659de07..ab8c465 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -26,8 +26,8 @@ import 
org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.StringSplitter;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
  * Table Metadata from Source. All name should be uppercase.
@@ -36,7 +36,9 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class TableDesc extends RootPersistentEntity implements ISourceAware {
 
-public static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
+private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
+private static final String materializedTableNamePrefix = 
"kylin_intermediate_";
+
 @JsonProperty("name")
 private String name;

[44/50] [abbrv] kylin git commit: KYLIN-2250 HiveMRInput accepts intermediate_view_table_name

2016-12-06 Thread lidong

KYLIN-2250 HiveMRInput accepts intermediate_view_table_name

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5c05880
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5c05880
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5c05880

Branch: refs/heads/master-hbase1.x
Commit: b5c058801b916ddc6e00085d652ad6a0497ce7d9
Parents: 7699973
Author: Cheng Wang 
Authored: Tue Dec 6 15:48:21 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 6 16:46:40 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeManager.java  |  2 +-
 .../apache/kylin/cube/cli/DictionaryGeneratorCLI.java |  2 +-
 .../org/apache/kylin/metadata/model/TableDesc.java| 14 +-
 .../org/apache/kylin/source/hive/HiveMRInput.java | 13 +
 4 files changed, 20 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 0c80c07..bda1423 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -274,7 +274,7 @@ public class CubeManager implements IRealizationProvider {
 SnapshotManager snapshotMgr = getSnapshotManager();
 
 TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable));
-if 
(TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+if (tableDesc.isView()) {
 String tableName = tableDesc.getMaterializedName();
 
tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable());
 tableDesc.setName(tableName);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 163c6ca..3e1ab0d 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -98,7 +98,7 @@ public class DictionaryGeneratorCLI {
 } else {
 MetadataManager metadataManager = 
MetadataManager.getInstance(config);
 TableDesc tableDesc = new 
TableDesc(metadataManager.getTableDesc(srcTable));
-if 
(TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+if (tableDesc.isView()) {
 TableDesc materializedTbl = new TableDesc();
 
materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
 materializedTbl.setName(tableDesc.getMaterializedName());

http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 659de07..ab8c465 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -26,8 +26,8 @@ import 
org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.StringSplitter;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
  * Table Metadata from Source. All name should be uppercase.
@@ -36,7 +36,9 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class TableDesc extends RootPersistentEntity implements ISourceAware {
 
-public static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
+private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
+private static final String materializedTableNamePrefix = 
"kylin_intermediate_";
+
 @JsonProperty("name")
 private String name;

[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class

[24/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global Dictionary

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
--
diff --git 
a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
 
b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
index caa8754..949599d 100644
--- 
a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
+++ 
b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
@@ -1,402 +1,402 @@
-2013-03-31,48028,0,\N,\N,\N,Auction,12,184.21,1001,1,Ebay
-2013-11-12,164262,0,\N,\N,\N,Others,5,172.03,1002,1,Ebay
-2013-04-06,82494,15,BookMagazines,NULL,Comic 
Books,Auction,14,66.6,1003,1,Ebay
-2013-05-17,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,1004,1,Ebay
-2013-05-20,152801,0,Jewelry & 
Watches,NULL,Earrings,FP-GTC,5,132.33,1005,1,Ebay
-2013-06-16,43398,0,Home & Garden,NULL,Cheese & 
Crackers,FP-GTC,13,7.12,1006,1,Ebay
-2013-06-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & 
Kits,Auction,14,204.28,1007,1,Ebay
-2013-03-22,158666,15,ToyHobbies,Action Figures,Anime & 
Manga,Auction,13,35.72,1008,1,Ebay
-2013-03-10,12688,0,eBay Premier,Books & Manuscripts,Books: 
Other,Auction,12,4.13,1009,1,Ebay
-2013-11-01,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed 
Items,FP-GTC,5,27.48,1010,1,Ebay
-2013-06-16,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & 
Cleaning,FP-GTC,14,9.26,1011,1,Ebay
-2013-09-12,80287,0,Computers/Tablets & Networking,Software,Office & 
Business,Auction,12,3.18,1012,1,Ebay
-2013-09-28,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck 
Parts,Others,13,3.18,1013,1,Ebay
-2013-06-15,87118,0,Sporting Goods,Outdoor 
Sports,Paintball,ABIN,14,377.94,1014,1,Ebay
-2013-03-14,25147,0,Sports MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Baseball-MLB,Auction,12,146.33,1015,1,Ebay
-2013-09-01,170302,15,Crafts,Embroidery,Design 
CDs,FP-GTC,5,51.23,1016,1,Ebay
-2013-05-29,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage 
Farm Equip,FP-non GTC,13,72.65,1017,1,Ebay
-2013-05-31,132939,0,Jewelry & Watches,Fashion 
Jewelry,Other,Auction,13,66.6,1018,1,Ebay
-2013-03-18,113593,15,Phones,Mobile Phones,Mobile 
Phones,Auction,12,9.26,1019,1,Ebay
-2013-07-19,34273,100,eBay Motors,Parts & 
Accessories,Motorcycle,Auction,14,583.44,1020,1,Ebay
-2013-06-23,106340,15,Home & Garden,Gardening,Hand 
Tools,FP-GTC,14,638.72,1021,1,Ebay
-2013-05-20,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,1022,1,Ebay
-2013-05-17,24760,0,Sports MeCards & Fan Shop,Fan Apparel & 
Souvenirs,Hockey-NHL,FP-GTC,12,319.79,1023,1,Ebay
-2013-03-11,37831,0,Collectibles,Advertising,Merchandise & 
Memorabilia,Auction,12,20.35,1024,1,Ebay
-2013-01-30,1120,3,Books,First Editions,Other,FP-non 
GTC,5,223.63,1025,1,Ebay
-2013-01-26,43972,100,eBay Motors,Parts & Accessories,ATV 
Parts,FP-GTC,13,204.28,1026,1,Ebay
-2013-03-22,166013,15,Computers,Computer Components & Parts,Video Capture & TV 
Tuner Cards,Auction,14,5.48,1027,1,Ebay
-2013-07-23,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,1028,1,Ebay
-2013-07-27,103178,15,ClothinShoeAccessories,Women's Bags,Women's 
Bags,FP-GTC,5,21.72,1029,1,Ebay
-2013-10-29,2023,0,Sporting Goods,Team 
Sports,Basketball,ABIN,12,3.18,1030,1,Ebay
-2013-10-08,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car 
Video,FP-GTC,11,491.32,1031,1,Ebay
-2013-04-26,15868,0,Real Estate,Land,Land,Auction,14,448.8,1032,1,Ebay
-2013-01-01,32876,0,Home & Garden,Home Improvement,Plumbing & 
Fixtures,Auction,13,415.73,1033,1,Ebay
-2013-01-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic 
Apparel,Auction,13,377.94,1034,1,Ebay
-2013-05-27,33038,15,Musical Instruments,Instruments,Guitars 
(Electric),FP-GTC,14,146.33,1035,1,Ebay
-2013-11-11,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: 
RacinNASCAR,FP-GTC,5,7.12,1036,1,Ebay
-2013-03-08,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & 
Conditioning,Auction,13,42.99,1037,1,Ebay
-2013-03-25,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic 
Apparel,Auction,13,12.85,1038,1,Ebay
-2013-08-20,15115,0,Video Games & Consoles,Video Games,Video 
Games,FP-GTC,13,55.89,1039,1,Ebay
-2013-05-17,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm 
Bracelets,FP-GTC,14,73.26,1040,1,Ebay
-2013-06-05,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & 
Vans,Auction,11,112.56,1041,1,Ebay
-2013-10-08,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio 
Accessories,FP-non GTC,11,3.49,1042,1,Ebay
-2013-08-14,121153,0,Baby,Nursery Decor,Night 
Lights,Auction,13,184.21,1043,1,Ebay
-2013-08-14,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & 
Laser Detectors,Auction,13,157.14,1044,1,Ebay

[30/50] [abbrv] kylin git commit: KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization

2016-12-06 Thread lidong

KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb686a94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb686a94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb686a94

Branch: refs/heads/master-hbase1.x
Commit: eb686a9428d930a60ec87883a2c253fb07f80a72
Parents: 0fd1ed6
Author: Li Yang 
Authored: Fri Dec 2 18:34:29 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 18:34:29 2016 +0800

--
 .../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb686a94/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 1e836b7..f6e687b 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -151,17 +151,21 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 @Override
 public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 BytesUtil.writeUTFString(name, buffer);
+buffer.put((byte) (isReversed ? 1 : 0));
 }
 
 @Override
 public void deserialize(IFilterCodeSystem cs, ByteBuffer buffer) {
 this.name = BytesUtil.readUTFString(buffer);
+this.isReversed = buffer.get() != 0;
 this.initMethod();
 }
 
 @Override
 public String toString() {
 StringBuilder sb = new StringBuilder();
+if (isReversed)
+sb.append("NOT ");
 sb.append(name);
 sb.append("(");
 for (int i = 0; i < methodParams.size(); i++) {

[02/50] [abbrv] kylin git commit: KYLIN-2213 minor code review

2016-12-06 Thread lidong

KYLIN-2213 minor code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2ecf18d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2ecf18d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2ecf18d

Branch: refs/heads/master-hbase1.x
Commit: a2ecf18de8ceb73f6b2748cd3e1d7b43ac1ca8ac
Parents: be7b5d1
Author: Li Yang 
Authored: Mon Nov 28 16:28:56 2016 +0800
Committer: Li Yang 
Committed: Mon Nov 28 16:28:56 2016 +0800

--
 .../apache/kylin/dict/BuiltInFunctionTransformer.java | 14 ++
 1 file changed, 2 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a2ecf18d/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 0f8e895..f451c2c 100755
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -89,12 +89,7 @@ public class BuiltInFunctionTransformer implements 
ITupleFilterTransformer {
 if (dict == null)
 return null;
 
-CompareTupleFilter translated;
-if (builtInFunctionTupleFilter.isReversed()) {
-translated = new CompareTupleFilter(FilterOperatorEnum.NOTIN);
-} else {
-translated = new CompareTupleFilter(FilterOperatorEnum.IN);
-}
+CompareTupleFilter translated = new 
CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? 
FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
 translated.addChild(new ColumnTupleFilter(columnRef));
 
 try {
@@ -126,12 +121,7 @@ public class BuiltInFunctionTransformer implements 
ITupleFilterTransformer {
 if (dict == null)
 return null;
 
-CompareTupleFilter translated;
-if (builtInFunctionTupleFilter.isReversed()) {
-translated = new CompareTupleFilter(FilterOperatorEnum.NOTIN);
-} else {
-translated = new CompareTupleFilter(FilterOperatorEnum.IN);
-}
+CompareTupleFilter translated = new 
CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? 
FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
 translated.addChild(new ColumnTupleFilter(columnRef));
 
 try {

[46/50] [abbrv] kylin git commit: KYLIN-2251: fix httpcore dependency conflict in JDBC Driver

2016-12-06 Thread lidong

KYLIN-2251: fix httpcore dependency conflict in JDBC Driver


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea13af4d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea13af4d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea13af4d

Branch: refs/heads/master-hbase1.x
Commit: ea13af4d8aa6a5b25fc328ee1adaf2b454eddea0
Parents: a53eafa
Author: Billy Liu 
Authored: Wed Dec 7 09:01:52 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 09:01:52 2016 +0800

--
 jdbc/pom.xml | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ea13af4d/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 4f42913..22d8eca 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -39,14 +39,14 @@
 
 
 
-org.apache.calcite.avatica
-avatica
-
-
 org.apache.httpcomponents
 httpclient
 
 
+org.apache.calcite.avatica
+avatica
+
+
 junit
 junit
 test

[45/50] [abbrv] kylin git commit: KYLIN-1971 ColumnDesc.equals() bug fix

2016-12-06 Thread lidong

KYLIN-1971 ColumnDesc.equals() bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a53eafaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a53eafaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a53eafaa

Branch: refs/heads/master-hbase1.x
Commit: a53eafaa30d5bb8bf5b52404bb6bbc1dcd9673e8
Parents: b5c0588
Author: Li Yang 
Authored: Tue Dec 6 18:39:03 2016 +0800
Committer: Li Yang 
Committed: Tue Dec 6 18:39:03 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a53eafaa/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 7d9133d..403eaaf 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -194,7 +194,7 @@ public class ColumnDesc implements Serializable {
 if (table == null) {
 if (other.table != null)
 return false;
-} else if (!table.equals(other.table))
+} else if (!table.getIdentity().equals(other.table.getIdentity()))
 return false;
 
 if (datatype == null) {

[43/50] [abbrv] kylin git commit: minor, generate item_count value for sample data

2016-12-06 Thread lidong

minor, generate item_count value for sample data

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2aaf270
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2aaf270
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2aaf270

Branch: refs/heads/master-hbase1.x
Commit: d2aaf2705a9035be9dbef8205fc9a1358b4316c2
Parents: 9968c22
Author: Yiming Liu 
Authored: Fri Dec 2 16:39:00 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 6 13:50:28 2016 +0800

--
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 1 file changed, 1 insertions(+), 1 deletions(-)
--

[16/50] [abbrv] kylin git commit: org.apache.kylin.metadata.model.FunctionDesc#configuration should use default linkedhashmap to keep order

2016-12-06 Thread lidong

org.apache.kylin.metadata.model.FunctionDesc#configuration should use default 
linkedhashmap to keep order


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/93bf0d02
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/93bf0d02
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/93bf0d02

Branch: refs/heads/master-hbase1.x
Commit: 93bf0d0214fe446d794b61e02fb18ff8097611d1
Parents: 30cb1ac
Author: Hongbin Ma 
Authored: Thu Dec 1 11:20:53 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 11:40:25 2016 +0800

--
 .../apache/kylin/common/util/JacksonBean.java| 16 
 .../apache/kylin/common/util/JacksonTest.java| 19 +++
 .../kylin/metadata/model/FunctionDesc.java   |  8 
 3 files changed, 35 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
index 42357f2..6d1 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -23,6 +23,10 @@ import 
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class JacksonBean {
 
@@ -32,6 +36,10 @@ public class JacksonBean {
 @JsonInclude(JsonInclude.Include.NON_NULL)
 private int b;
 
+@JsonProperty("configuration")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+private Map configuration = new HashMap();
+
 public String getA() {
 return a;
 }
@@ -48,6 +56,14 @@ public class JacksonBean {
 this.b = b;
 }
 
+public Map getConfiguration() {
+return configuration;
+}
+
+public void setConfiguration(Map configuration) {
+this.configuration = configuration;
+}
+
 @Override
 public String toString() {
 return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';

http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
index 81be7eb..0d7097a 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -18,22 +18,33 @@
 
 package org.apache.kylin.common.util;
 
-import java.io.IOException;
-
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 public class JacksonTest {
 @Test
 public void foo() throws IOException {
+HashMap a = new HashMap();
+a.put("1", "1");
+a.put("3", "3");
+a.put("2", "2");
+
+
 JacksonBean bean = new JacksonBean();
 bean.setA("valuea");
+bean.setConfiguration(a);
 
 String s = JsonUtil.writeValueAsString(bean);
 System.out.println(s);
 
-JacksonBean desBean = (JacksonBean) 
JsonUtil.readValue("{\"a\":\"valuea\"}", JacksonBean.class);
+JacksonBean desBean = (JacksonBean) 
JsonUtil.readValue("{\"a\":\"valuea\",\"b\":0,\"configuration\":{\"2\":\"2\",\"3\":\"3\",\"1\":\"1\"}}",
 JacksonBean.class);
+
 String x2 = JsonUtil.writeValueAsString(desBean);
-System.out.println(desBean);
 System.out.println(x2);
+
+System.out.println(desBean);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index ae7f805..7b41552 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++

[31/50] [abbrv] kylin git commit: minor: normalize scientific decimal values

2016-12-06 Thread lidong

minor: normalize scientific decimal values


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fe9efa86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fe9efa86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fe9efa86

Branch: refs/heads/master-hbase1.x
Commit: fe9efa86cd34da31ec41dc176140ee4ee2f49fed
Parents: eb686a9
Author: Hongbin Ma 
Authored: Sun Dec 4 21:26:18 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Dec 4 21:26:18 2016 +0800

--
 .../java/org/apache/kylin/metadata/tuple/Tuple.java | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fe9efa86/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index 721a719..d51ae9e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -115,6 +115,8 @@ public class Tuple implements ITuple {
 // BigDecimal during cube build for best precision
 if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
 fieldValue = ((BigDecimal) fieldValue).doubleValue();
+} else if ("decimal".equals(dataType) && fieldValue instanceof 
BigDecimal) {
+fieldValue = normalizeDecimal((BigDecimal) fieldValue);
 } else if ("integer".equals(dataType) && fieldValue instanceof Number) 
{
 fieldValue = ((Number) fieldValue).intValue();
 } else if ("smallint".equals(dataType) && fieldValue instanceof 
Number) {
@@ -145,6 +147,14 @@ public class Tuple implements ITuple {
 return o;
 }
 
+private static BigDecimal normalizeDecimal(BigDecimal input) {
+if (input.scale() < 0) {
+return input.setScale(0);
+} else {
+return input;
+}
+}
+
 public boolean hasColumn(TblColRef column) {
 return info.hasColumn(column);
 }
@@ -199,7 +209,7 @@ public class Tuple implements ITuple {
 case "double":
 return Double.valueOf(strValue);
 case "decimal":
-return new BigDecimal(strValue);
+return normalizeDecimal(new BigDecimal(strValue));
 case "float":
 return Float.valueOf(strValue);
 case "boolean":

[38/50] [abbrv] kylin git commit: KYLIN-2245 slim Segments in CubeMananger

2016-12-06 Thread lidong

KYLIN-2245 slim Segments in CubeMananger

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a441c3f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a441c3f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a441c3f

Branch: refs/heads/master-hbase1.x
Commit: 0a441c3fa30aee0a06cfc6301f7fbfa412103179
Parents: 59a30f6
Author: Cheng Wang 
Authored: Mon Dec 5 16:17:31 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 20:20:30 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |  14 ++
 .../java/org/apache/kylin/cube/CubeManager.java | 165 +--
 .../apache/kylin/metadata/model/ISegment.java   |   5 +-
 .../apache/kylin/metadata/model/Segments.java   | 206 ++-
 4 files changed, 221 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 8b12c2e..ecbb437 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.cube;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Set;
 
@@ -25,6 +26,7 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -361,6 +363,18 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return this.getDescriptor().getAutoMergeTimeRanges() != null && 
this.getDescriptor().getAutoMergeTimeRanges().length > 0;
 }
 
+public Pair autoMergeCubeSegments() throws IOException {
+return segments.autoMergeCubeSegments(needAutoMerge(), getName(), 
getDescriptor().getAutoMergeTimeRanges());
+}
+
+public Segments calculateToBeSegments(CubeSegment newSegment) {
+return segments.calculateToBeSegments(newSegment, 
getModel().getPartitionDesc().isPartitioned());
+}
+
+public Pair 
findMergeOffsetsByDateRange(Segments segs, long startDate, long 
endDate, long skipSegDateRangeCap) {
+return this.segments.findMergeOffsetsByDateRange(segs, startDate, 
endDate, skipSegDateRangeCap);
+}
+
 public CubeSegment getLastSegment() {
 List existing = getSegments();
 if (existing.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4ba29af..296a4e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -520,7 +519,7 @@ public class CubeManager implements IRealizationProvider {
 if (isOffsetsOn) {
 // offset cube, merge by date range?
 if (startOffset == endOffset) {
-Pair pair = 
findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
+Pair pair = 
cube.findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
 if (pair == null)
 throw new IllegalArgumentException("Find no segments to 
merge by date range " + startDate + "-" + endDate + " for cube " + cube);
 startOffset = pair.getFirst().getSourceOffsetStart();
@@ -580,32 +579,6 @@ public class CubeManager implements IRealizationProvider {
 return newSegment;
 }
 
-private Pair 
findMergeOffsetsByDateRange(List segments, long startDate, long 
endDate, long skipSegDateRangeCap) {
-// must be offset cube
-

[04/50] [abbrv] kylin git commit: minor, fix typo

2016-12-06 Thread lidong

minor, fix typo


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3d62e9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3d62e9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3d62e9c

Branch: refs/heads/master-hbase1.x
Commit: c3d62e9c5e6c39771bec1bbe2660130ff2033ef3
Parents: b1448e5
Author: lidongsjtu 
Authored: Mon Nov 28 23:48:46 2016 +0800
Committer: lidongsjtu 
Committed: Mon Nov 28 23:48:46 2016 +0800

--
 build/smoke-test/testDiag.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c3d62e9c/build/smoke-test/testDiag.py
--
diff --git a/build/smoke-test/testDiag.py b/build/smoke-test/testDiag.py
index cc932da..588864b 100644
--- a/build/smoke-test/testDiag.py
+++ b/build/smoke-test/testDiag.py
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# This is python unittest used in smoke-test.sh, aim to testing query via rest 
APIs.
+# This is python unittest used in smoke-test.sh, aim to testing diagnosis via 
rest APIs.
 
 import unittest
 import requests
@@ -40,5 +40,5 @@ class testDiag(unittest.TestCase):
 
 
 if __name__ == '__main__':
-print 'Test Diagnogis for Kylin sample.'
+print 'Test Diagnosis for Kylin sample.'
 unittest.main()

[14/50] [abbrv] kylin git commit: KYLIN-2217 Reducers build dictionaries locally

2016-12-06 Thread lidong

KYLIN-2217 Reducers build dictionaries locally

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1af08e4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1af08e4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1af08e4b

Branch: refs/heads/master-hbase1.x
Commit: 1af08e4b8875d33bfc5dd124fed72d6042456c32
Parents: b1b90ad
Author: xiefan46 <958034...@qq.com>
Authored: Wed Nov 23 09:48:55 2016 +0800
Committer: Li Yang 
Committed: Wed Nov 30 15:31:58 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   9 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   3 +-
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  |  25 ++-
 .../apache/kylin/dict/DictionaryManager.java|  17 +-
 .../apache/kylin/dict/DictionaryProvider.java   |  28 
 .../dict/DictionaryReducerLocalGenerator.java   | 156 +++
 .../dict/IDictionaryReducerLocalBuilder.java|  31 
 .../kylin/dict/DictionaryProviderTest.java  | 109 +
 .../storage/translate/ColumnValueRange.java |   2 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  44 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 123 +--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  54 +++
 .../storage/hbase/cube/v1/CubeStorageQuery.java |   6 +-
 13 files changed, 547 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7dcc771..766c04d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -719,7 +719,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 //UHC: ultra high cardinality columns, contain the ShardByColumns and the 
GlobalDictionaryColumns
 public int getUHCReducerCount() {
-return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
+return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
+}
+
+public boolean isReducerLocalBuildDict() {
+if (getUHCReducerCount() != 1) {
+return false;
+}
+return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
 }
 
 public String getYarnStatusCheckUrl() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b4422d2..119a21a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -213,6 +213,7 @@ public class CubeManager implements IRealizationProvider {
 return result;
 }
 
+
 public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, 
ReadableTable inpTable) throws IOException {
 CubeDesc cubeDesc = cubeSeg.getCubeDesc();
 if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
@@ -221,6 +222,7 @@ public class CubeManager implements IRealizationProvider {
 String builderClass = cubeDesc.getDictionaryBuilderClass(col);
 DictionaryInfo dictInfo = 
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, 
builderClass);
 
+
 saveDictionaryInfo(cubeSeg, col, dictInfo);
 return dictInfo;
 }
@@ -266,7 +268,6 @@ public class CubeManager implements IRealizationProvider {
 } catch (IOException e) {
 throw new IllegalStateException("Failed to get dictionary for cube 
segment" + cubeSeg + ", col" + col, e);
 }
-
 return (Dictionary) info.getDictionaryObject();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index a6aeb96..a4e1df0 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++

[41/50] [abbrv] kylin git commit: KYLIN-2246 redesign the way to decide layer cubing reducer count

2016-12-06 Thread lidong

KYLIN-2246 redesign the way to decide layer cubing reducer count


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76999735
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76999735
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76999735

Branch: refs/heads/master-hbase1.x
Commit: 76999735f4157b6064eb099b59a7c79fbc9b6007
Parents: d2aaf27
Author: Hongbin Ma 
Authored: Mon Dec 5 21:02:36 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Dec 6 13:50:28 2016 +0800

--
 .../kylin/cube/cuboid/CuboidScheduler.java  | 31 +++-
 .../kylin/engine/mr/common/CubeStatsReader.java | 26 ++-
 .../apache/kylin/engine/mr/steps/CuboidJob.java | 52 +
 .../engine/mr/steps/LayerReduerNumSizing.java   | 82 
 .../kylin/engine/mr/steps/MergeCuboidJob.java   |  2 +-
 5 files changed, 138 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/76999735/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index bd6a37a..733aded 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -18,7 +18,7 @@
 
 package org.apache.kylin.cube.cuboid;
 
-/** 
+/**
  */
 
 import java.util.Collections;
@@ -31,6 +31,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import org.apache.kylin.cube.model.AggregationGroup;
 import org.apache.kylin.cube.model.CubeDesc;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
@@ -39,6 +40,7 @@ public class CuboidScheduler {
 private final CubeDesc cubeDesc;
 private final long max;
 private final Map cache;
+private List cuboidsByLayer;
 
 public CuboidScheduler(CubeDesc cubeDesc) {
 this.cubeDesc = cubeDesc;
@@ -232,4 +234,31 @@ public class CuboidScheduler {
 getSubCuboidIds(cuboidId, result);
 }
 }
+
+public List getCuboidsByLayer() {
+if (cuboidsByLayer != null) {
+return cuboidsByLayer;
+}
+
+int totalNum = 0;
+int layerNum = cubeDesc.getBuildLevel();
+cuboidsByLayer = Lists.newArrayList();
+
+
cuboidsByLayer.add(Collections.singletonList(Cuboid.getBaseCuboidId(cubeDesc)));
+totalNum++;
+
+for (int i = 1; i <= layerNum; i++) {
+List lastLayer = cuboidsByLayer.get(i - 1);
+List newLayer = Lists.newArrayList();
+for (Long parent : lastLayer) {
+newLayer.addAll(getSpanningCuboid(parent));
+}
+cuboidsByLayer.add(newLayer);
+totalNum += newLayer.size();
+}
+
+int size = getAllCuboidIds().size();
+Preconditions.checkState(totalNum == size, "total Num: " + totalNum + 
" actual size: " + size);
+return cuboidsByLayer;
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/76999735/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c917cfb..1cf5da6 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -29,6 +29,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BytesWritable;
@@ -75,9 +76,11 @@ public class CubeStatsReader {
 final int mapperNumberOfFirstBuild; // becomes meaningless after merge
 final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after 
merge
 final Map cuboidRowEstimatesHLL;
+final CuboidScheduler cuboidScheduler;
 
 public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) 
throws IOException {
 ResourceStore store = ResourceStore.getStore(kylinConfig);
+cuboidScheduler = new CuboidScheduler(cubeSegment.getCubeDesc());
 String statsKey = cubeSegment.getStatisticsResourcePath();
 File tmpSeqFile =

[25/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global Dictionary

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java 
b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
index df64f3f..381e0b1 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
@@ -17,11 +17,12 @@
 */
 package org.apache.kylin.dict;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.*;
@@ -91,39 +92,26 @@ public class CachedTreeMapTest {
 public static class CachedFileFilter implements FileFilter {
 @Override
 public boolean accept(File pathname) {
-return pathname.getName().startsWith("cached_");
+return pathname.getName().startsWith(CachedTreeMap.CACHED_PREFIX);
 }
 }
 
-public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
-public static final String backupDir = 
"/tmp/kylin_cachedtreemap_test.bak/";
-public static final String tmpDir = "/tmp/kylin_cachedtreemap_test.tmp/";
-
-private static void cleanup() {
-File dir = new File(baseDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
-}
-
-dir = new File(tmpDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
+public static class VersionFilter implements FileFilter {
+@Override
+public boolean accept(File pathname) {
+return pathname.getName().startsWith(CachedTreeMap.VERSION_PREFIX);
 }
+}
 
-dir = new File(backupDir);
-if (dir.exists()) {
-for (File f : dir.listFiles()) {
-f.delete();
-}
-dir.delete();
-}
+public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
+public static final String workingDir = 
"/tmp/kylin_cachedtreemap_test/working";
 
+private static void cleanup() {
+Configuration conf = new Configuration();
+Path basePath = new Path(baseDir);
+try {
+FileSystem.get(basePath.toUri(), conf).delete(basePath, true);
+} catch (IOException e) {}
 VALUE_WRITE_ERROR_TOGGLE = false;
 }
 
@@ -139,154 +127,240 @@ public class CachedTreeMapTest {
 
 @Test
 public void testCachedTreeMap() throws IOException {
-CachedTreeMap map = 
CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
-
.persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
+CachedTreeMap map = createMutableMap();
 map.put(Key.of(1), Value.of("a"));
 map.put(Key.of(2), Value.of("b"));
 map.put(Key.of(3), Value.of("c"));
 map.put(Key.of(4), Value.of("d"));
 map.put(Key.of(5), Value.of("e"));
 
-File dir = new File(tmpDir);
+File dir = new File(workingDir);
 assertEquals(3, dir.listFiles(new CachedFileFilter()).length);
 
-DataOutputStream out = new DataOutputStream(new 
FileOutputStream(tmpDir+"/.index"));
-map.write(out);
-out.flush();
-out.close();
-map.commit(false);
+flushAndCommit(map, true, true, false);
+assertFalse(new File(workingDir).exists());
 
-dir = new File(baseDir);
+dir = new File(map.getLatestVersion());
 assertEquals(5, dir.listFiles(new CachedFileFilter()).length);
 
-DataInputStream in = new DataInputStream(new 
FileInputStream(baseDir+".index"));
-CachedTreeMap map2 = 
CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
-
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
-map2.readFields(in);
+CachedTreeMap map2 = createImmutableMap();
 assertEquals(5, map2.size());
 assertEquals("b", ((Value)map2.get(Key.of(2))).valueStr);
 
 try {
 map2.put(Key.of(6), Value.of("f"));
 fail("Should be error when put value into immutable map");
-} catch (AssertionError error) {
+} catch (AssertionError error) {}
+}
+
+@Test
+public void testMultiVersions() throws IOException, InterruptedException {
+CachedTreeMap map = createMutableMap();
+Thread.sleep(3000);
+map.put(Key.of(1), Value.of("a"));
+map.put(Key.of(2),

[22/50] [abbrv] kylin git commit: KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally

2016-12-06 Thread lidong

KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e562aafe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e562aafe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e562aafe

Branch: refs/heads/master-hbase1.x
Commit: e562aafee10c8bbd125937503d42b958c1843aad
Parents: 3091f06
Author: Hongbin Ma 
Authored: Fri Dec 2 13:33:02 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Dec 2 13:33:18 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java   | 18 ++
 .../java/org/apache/kylin/jdbc/IRemoteClient.java |  3 ++-
 .../java/org/apache/kylin/jdbc/KylinClient.java   |  7 ---
 .../org/apache/kylin/jdbc/KylinResultSet.java |  9 -
 .../org/apache/kylin/jdbc/json/QueryRequest.java  | 12 
 .../java/org/apache/kylin/jdbc/DummyClient.java   |  3 ++-
 .../apache/kylin/rest/service/QueryService.java   | 11 ++-
 7 files changed, 56 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index f7c90aa..28f7697 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -79,6 +79,14 @@ public class BackdoorToggles {
 }
 }
 
+public static Integer getStatementMaxRows() {
+String v = getString(ATTR_STATEMENT_MAX_ROWS);
+if (v == null)
+return null;
+else
+return Integer.valueOf(v);
+}
+
 private static String getString(String key) {
 Map toggles = _backdoorToggles.get();
 if (toggles == null) {
@@ -183,4 +191,14 @@ public class BackdoorToggles {
  */
 public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = 
"DEBUG_TOGGLE_SHARD_ASSIGNMENT";
 
+// properties on statement may go with this "channel" too
+/**
+ * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "ATTR_STATEMENT_MAX_ROWS": "10"
+ }
+ */
+public final static String ATTR_STATEMENT_MAX_ROWS = 
"ATTR_STATEMENT_MAX_ROWS";
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
index b6a13e5..dfd8d76 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.calcite.avatica.AvaticaParameter;
 import org.apache.calcite.avatica.ColumnMetaData;
@@ -51,6 +52,6 @@ public interface IRemoteClient extends Closeable {
 /**
  * Execute query remotely and get back result.
  */
-public QueryResult executeQuery(String sql, List params, 
List paramValues) throws IOException;
+public QueryResult executeQuery(String sql, List params, 
List paramValues, Map queryToggles) throws IOException;
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index 2d06a92..86c3a5b 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -321,9 +321,9 @@ public class KylinClient implements IRemoteClient {
 }
 
 @Override
-public QueryResult executeQuery(String sql, List params, 
List paramValues) throws IOException {
+public QueryResult executeQuery(String sql, List params, 
List paramValues, Map queryToggles) throws IOException {
 
-SQLResponseStub queryResp = executeKylinQuery(sql, 
convertParameters(params, paramValues));
+SQLResponseStub queryResp = executeKylinQuery(sql, 
convertParameters(params, paramValues), queryToggles);
 if

[03/50] [abbrv] kylin git commit: KYLIN-2195 support backward-compatibility properties in get-properties.sh

2016-12-06 Thread lidong

KYLIN-2195 support backward-compatibility properties in get-properties.sh


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1448e57
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1448e57
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1448e57

Branch: refs/heads/master-hbase1.x
Commit: b1448e5789ce65d5b94ed246df01fa4c269515b2
Parents: a2ecf18
Author: lidongsjtu 
Authored: Mon Nov 28 23:37:41 2016 +0800
Committer: lidongsjtu 
Committed: Mon Nov 28 23:37:41 2016 +0800

--
 build/bin/get-properties.sh |  4 +-
 .../org/apache/kylin/tool/KylinConfigCLI.java   | 42 
 2 files changed, 45 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b1448e57/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 1a086ea..170442d 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -25,5 +25,7 @@ then
 exit -1
 fi
 
-result=`cat ${KYLIN_HOME}/conf/kylin.properties | grep -w "^$1" | grep -v '^#' 
| awk -F= '{ n = index($0,"="); print substr($0,n+1)}' | cut -c 1- |tail -1`
+job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 
2>/dev/null`
 echo "$result"

http://git-wip-us.apache.org/repos/asf/kylin/blob/b1448e57/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
new file mode 100644
index 000..e1a5b99
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool;
+
+import java.util.Properties;
+
+import org.apache.kylin.common.BackwardCompatibilityConfig;
+import org.apache.kylin.common.KylinConfig;
+
+public class KylinConfigCLI {
+public static void main(String[] args) {
+if (args.length != 1) {
+System.err.println("Usage: KylinConfigCLI conf_name");
+System.err.println("Example: KylinConfigCLI kylin.server.mode");
+System.exit(1);
+}
+
+Properties config = KylinConfig.getKylinProperties();
+BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig();
+String value = config.getProperty(bcc.check(args[0]));
+if (value == null) {
+value = "";
+}
+System.out.println(value);
+}
+}

[15/50] [abbrv] kylin git commit: KYLIN-2217 Code review, refactor IDictionaryBuilder

2016-12-06 Thread lidong

KYLIN-2217 Code review, refactor IDictionaryBuilder


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30cb1ac6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30cb1ac6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30cb1ac6

Branch: refs/heads/master-hbase1.x
Commit: 30cb1ac678b063ac164acd27591e8a1d0becafa0
Parents: 1af08e4
Author: Li Yang 
Authored: Wed Nov 30 15:30:15 2016 +0800
Committer: Yang Li 
Committed: Wed Nov 30 21:00:17 2016 +0800

--
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  |   2 +-
 .../apache/kylin/dict/DictionaryGenerator.java  | 165 ---
 .../apache/kylin/dict/DictionaryManager.java|  16 +-
 .../apache/kylin/dict/DictionaryProvider.java   |   4 +-
 .../dict/DictionaryReducerLocalGenerator.java   | 156 --
 .../kylin/dict/GlobalDictionaryBuilder.java |  36 ++--
 .../apache/kylin/dict/IDictionaryBuilder.java   |  13 +-
 .../dict/IDictionaryReducerLocalBuilder.java|  31 
 .../kylin/dict/DictionaryProviderTest.java  |  63 +++
 .../engine/mr/steps/CreateDictionaryJob.java|  42 ++---
 .../mr/steps/FactDistinctColumnsReducer.java| 115 +
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   9 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   6 +-
 13 files changed, 231 insertions(+), 427 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index a4e1df0..163c6ca 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -60,7 +60,7 @@ public class DictionaryGeneratorCLI {
 for (TblColRef col : 
cubeSeg.getCubeDesc().getAllColumnsNeedDictionaryBuilt()) {
 logger.info("Building dictionary for " + col);
 ReadableTable inpTable = decideInputTable(cubeSeg.getModel(), col, 
factTableValueProvider);
-if (config.isReducerLocalBuildDict() && dictProvider != null) {
+if (dictProvider != null) {
 Dictionary dict = dictProvider.getDictionary(col);
 if (dict != null) {
 cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);

http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 810a392..cd13d59 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -19,12 +19,11 @@
 package org.apache.kylin.dict;
 
 import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.slf4j.Logger;
@@ -40,9 +39,7 @@ public class DictionaryGenerator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
-private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
-
-public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
+public static IDictionaryBuilder newDictionaryBuilder(DataType dataType) {
 Preconditions.checkNotNull(dataType, "dataType cannot be null");
 
 // build dict, case by data type
@@ -57,16 +54,33 @@ public class DictionaryGenerator {
 } else {
 builder = new StringDictBuilder();
 }
+return builder;
+}
 
-return buildDictionary(builder, null, valueEnumerator);
+public static Dictionary buildDictionary(DataType dataType, 
IDictionaryValueEnumerator valueEnumerator) throws IOException {
+return buildDictionary(newDictionaryBuilder(dataType), null, 
valueEnumerator);
 }
 
-public static Dictionary buildDictionary(IDictionaryBuilder 
builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) 
throws IOException {
+static

[42/50] [abbrv] kylin git commit: minor, generate item_count value for sample data

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/d2aaf270/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index 7349b37..e13b7ad 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,1 +1,1 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,1349,10002313,ANALYST,Beijing
-1,2012-08-28,Others,175750,0,13,23.8563,0,1927,10004376,ANALYST,Beijing
-2,2012-02-16,ABIN,148324,15,13,88.3418,0,1005,10006710,ADMIN,Shanghai
-3,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,1209,10003717,ANALYST,Beijing
-4,2012-10-22,Others,140746,100,11,83.454,0,1154,10006076,ADMIN,Shanghai
-5,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,1372,10007436,ADMIN,Shanghai
-6,2013-04-04,Others,963,0,13,88.5907,0,1648,10009869,MODELER,Hongkong
-7,2012-04-11,Others,15687,0,15,88.194,0,1866,1400,ADMIN,Shanghai
-8,2013-01-19,ABIN,60606,3,13,77.9727,0,1936,1687,MODELER,Hongkong
-9,2012-04-30,FP-non GTC,106246,0,14,52.295,0,1047,10009223,MODELER,Hongkong
-10,2013-02-03,Auction,45333,0,16,56.3584,0,1470,10006759,MODELER,Hongkong
-11,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,1203,1062,ANALYST,Beijing
-12,2013-07-30,Others,24760,0,16,25.077,0,1382,10003539,ADMIN,Shanghai
-13,2012-04-17,Auction,31519,0,16,72.7384,0,1069,10001245,ADMIN,Shanghai
-14,2013-04-30,FP-GTC,51582,0,15,75.82,0,1085,10008148,ANALYST,Beijing
-15,2013-03-23,FP-GTC,16509,0,15,8.6653,0,1559,10008079,MODELER,Hongkong
-16,2013-04-01,ABIN,175750,3,16,64.2802,0,1507,10007797,MODELER,Hongkong
-17,2013-07-12,ABIN,50677,0,-99,24.5987,0,1938,10006224,ADMIN,Shanghai
-18,2012-04-23,FP-GTC,45333,0,5,72.6553,0,1863,10003824,MODELER,Hongkong
-19,2012-12-19,ABIN,3838,0,12,87.4142,0,1612,10007157,ADMIN,Shanghai
-20,2012-08-15,FP-GTC,62179,0,16,67.4238,0,1839,10003452,MODELER,Hongkong
-21,2013-08-22,Auction,1357,0,5,18.36,0,1303,1114,ANALYST,Beijing
-22,2013-08-23,ABIN,31519,0,5,67.6344,0,1877,10007417,ANALYST,Beijing
-23,2013-09-12,FP-GTC,223,0,14,71.7898,0,1848,1627,MODELER,Hongkong
-24,2012-03-14,FP-GTC,132939,0,15,26.1496,0,1815,10005292,MODELER,Hongkong
-25,2013-07-13,Others,67703,3,16,83.9661,0,1858,10008123,MODELER,Hongkong
-26,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,1387,10007806,ADMIN,Shanghai
-27,2012-08-30,Auction,62179,0,11,21.0496,0,1393,10007360,ANALYST,Beijing
-28,2013-01-10,FP-GTC,48027,0,13,23.4142,0,1757,10002135,MODELER,Hongkong
-29,2013-01-25,ABIN,145970,0,5,26.7842,0,1402,10009682,ANALYST,Beijing
-30,2012-07-05,Auction,48027,0,-99,46.732,0,1972,10009522,MODELER,Hongkong
-31,2012-11-07,FP-GTC,20213,0,-99,19.727,0,1873,10009944,ANALYST,Beijing
-32,2013-06-20,ABIN,100847,0,5,76.2783,0,1409,10005395,ADMIN,Shanghai
-33,2012-10-08,ABIN,26262,0,5,48.7447,0,1029,10001836,ADMIN,Shanghai
-34,2013-04-30,ABIN,87118,0,15,63.2187,0,1529,1928,ANALYST,Beijing
-35,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,1329,1768,ANALYST,Beijing
-36,2012-05-13,Others,164261,0,13,84.1246,0,1688,1727,ADMIN,Shanghai
-37,2013-08-02,ABIN,31519,0,5,37.1504,0,1905,1719,MODELER,Hongkong
-38,2012-04-03,Auction,164261,0,11,92.0974,0,1341,10001337,ADMIN,Shanghai
-39,2012-09-02,FP-GTC,31519,0,14,87.7829,0,1460,10008484,ANALYST,Beijing
-40,2013-10-05,ABIN,13987,0,16,74.0719,0,1381,10002975,MODELER,Hongkong
-41,2012-01-13,FP-GTC,48027,0,15,71.2637,0,1915,10005778,ADMIN,Shanghai
-42,2013-03-01,Others,13836,0,14,16.7288,0,1953,10009385,ADMIN,Shanghai
-43,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,1277,10002286,MODELER,Hongkong
-44,2012-02-07,Auction,45333,0,5,64.977,0,1040,10005583,MODELER,Hongkong
-45,2013-12-14,FP-non 
GTC,158798,0,16,72.4413,0,1500,10007635,MODELER,Hongkong
-46,2013-10-13,Auction,31519,0,5,79.3053,0,1816,10006423,MODELER,Hongkong
-47,2012-06-11,ABIN,4943,0,5,11.6942,0,1696,10001237,ANALYST,Beijing
-48,2012-10-18,ABIN,80053,0,-99,54.0933,0,1604,10006851,MODELER,Hongkong
-49,2012-11-03,Auction,20886,0,5,9.8258,0,1764,10001220,MODELER,Hongkong
-50,2012-01-15,Auction,44079,0,14,13.0371,0,1343,10001259,MODELER,Hongkong
-51,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,1724,10009008,ADMIN,Shanghai
-52,2012-02-16,FP-GTC,314,0,13,49.1825,0,1334,10004005,ADMIN,Shanghai
-53,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,1603,10002595,ANALYST,Beijing
-54,2012-07-15,FP-non 
GTC,32876,0,-99,50.9634,0,1248,10006890,ANALYST,Beijing
-55,2012-07-24,FP-GTC,150265,15,16,57.5645,0,1653,10007269,ADMIN,Shanghai
-56,2012-10-06,Auction,145970,0,5,44.3091,0,1082,10001640,ADMIN,Shanghai
-57,2013-06-14,ABIN,80053,0,5,20.2388,0,1632,10002068,MODELER,Hongkong
-58,2013-10-10,FP-non

[07/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
--
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv 
b/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
new file mode 100644
index 000..253afbf
--- /dev/null
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
@@ -0,0 +1,1 @@
+1000,1,4,FR,N/A
+1001,0,1,DE,N/A
+1002,0,1,IT,N/A
+1003,4,2,DE,N/A
+1004,2,3,JP,N/A
+1005,0,1,CN,N/A
+1006,2,3,JP,N/A
+1007,0,2,GB,N/A
+1008,5,3,US,N/A
+1009,1,3,US,N/A
+1010,1,1,IT,N/A
+1011,2,4,FR,N/A
+1012,3,1,CN,N/A
+1013,1,1,FR,N/A
+1014,5,3,CN,N/A
+1015,3,5,IT,N/A
+1016,4,5,JP,N/A
+1017,3,4,US,N/A
+1018,0,5,CN,N/A
+1019,4,4,GB,N/A
+1020,4,4,IT,N/A
+1021,1,2,FR,N/A
+1022,2,2,DE,N/A
+1023,0,1,GB,N/A
+1024,3,4,IT,N/A
+1025,4,4,JP,N/A
+1026,5,4,IT,N/A
+1027,2,4,FR,N/A
+1028,2,2,GB,N/A
+1029,1,1,GB,N/A
+1030,5,5,CN,N/A
+1031,4,2,CN,N/A
+1032,1,3,CN,N/A
+1033,2,1,JP,N/A
+1034,2,1,JP,N/A
+1035,4,2,US,N/A
+1036,5,4,DE,N/A
+1037,2,4,DE,N/A
+1038,1,3,US,N/A
+1039,3,5,US,N/A
+1040,2,3,GB,N/A
+1041,1,5,IT,N/A
+1042,5,5,CN,N/A
+1043,5,1,IT,N/A
+1044,3,4,IT,N/A
+1045,1,3,US,N/A
+1046,3,3,JP,N/A
+1047,2,2,FR,N/A
+1048,5,4,JP,N/A
+1049,3,1,JP,N/A
+1050,3,5,JP,N/A
+1051,5,1,US,N/A
+1052,3,5,DE,N/A
+1053,1,3,JP,N/A
+1054,5,1,GB,N/A
+1055,2,1,GB,N/A
+1056,2,3,CN,N/A
+1057,1,5,US,N/A
+1058,2,2,CN,N/A
+1059,2,4,FR,N/A
+1060,0,3,FR,N/A
+1061,0,5,FR,N/A
+1062,1,1,US,N/A
+1063,4,5,DE,N/A
+1064,2,5,GB,N/A
+1065,0,3,GB,N/A
+1066,1,1,IT,N/A
+1067,0,2,GB,N/A
+1068,4,1,IT,N/A
+1069,0,4,FR,N/A
+1070,3,4,CN,N/A
+1071,3,5,GB,N/A
+1072,4,3,CN,N/A
+1073,0,4,IT,N/A
+1074,5,3,GB,N/A
+1075,0,4,JP,N/A
+1076,2,5,JP,N/A
+1077,0,3,CN,N/A
+1078,4,3,GB,N/A
+1079,4,5,US,N/A
+1080,4,2,JP,N/A
+1081,5,2,CN,N/A
+1082,4,1,JP,N/A
+1083,0,4,GB,N/A
+1084,1,4,DE,N/A
+1085,4,4,FR,N/A
+1086,2,5,JP,N/A
+1087,0,1,IT,N/A
+1088,0,2,FR,N/A
+1089,3,2,DE,N/A
+1090,5,2,IT,N/A
+1091,0,3,GB,N/A
+1092,1,4,DE,N/A
+1093,2,1,IT,N/A
+1094,4,3,JP,N/A
+1095,1,1,DE,N/A
+1096,0,5,JP,N/A
+1097,3,3,CN,N/A
+1098,5,3,JP,N/A
+1099,3,1,GB,N/A
+1100,5,1,CN,N/A
+1101,2,3,JP,N/A
+1102,5,1,IT,N/A
+1103,5,2,GB,N/A
+1104,3,5,GB,N/A
+1105,3,1,DE,N/A
+1106,3,5,GB,N/A
+1107,5,3,DE,N/A
+1108,4,1,CN,N/A
+1109,4,3,GB,N/A
+1110,4,1,FR,N/A
+1111,3,5,FR,N/A
+1112,1,1,GB,N/A
+1113,5,4,FR,N/A
+1114,2,2,DE,N/A
+1115,4,3,GB,N/A
+1116,0,3,FR,N/A
+1117,1,4,CN,N/A
+1118,4,4,IT,N/A
+1119,0,2,CN,N/A
+1120,5,2,DE,N/A
+1121,0,2,GB,N/A
+1122,4,3,CN,N/A
+1123,1,4,FR,N/A
+1124,0,2,CN,N/A
+1125,4,5,US,N/A
+1126,0,4,GB,N/A
+1127,4,3,CN,N/A
+1128,4,3,GB,N/A
+1129,1,4,JP,N/A
+1130,2,2,JP,N/A
+1131,0,3,IT,N/A
+1132,4,1,GB,N/A
+1133,1,5,US,N/A
+1134,3,3,DE,N/A
+1135,3,2,IT,N/A
+1136,2,2,JP,N/A
+1137,0,3,CN,N/A
+1138,3,1,FR,N/A
+1139,2,3,FR,N/A
+1140,3,1,IT,N/A
+1141,0,3,US,N/A
+1142,3,4,FR,N/A
+1143,4,5,DE,N/A
+1144,3,3,JP,N/A
+1145,5,5,DE,N/A
+1146,2,5,FR,N/A
+1147,0,5,IT,N/A
+1148,2,5,US,N/A
+1149,0,3,CN,N/A
+1150,2,2,GB,N/A
+1151,5,5,CN,N/A
+1152,2,2,IT,N/A
+1153,3,2,FR,N/A
+1154,5,4,FR,N/A
+1155,3,5,FR,N/A
+1156,5,2,GB,N/A
+1157,1,2,IT,N/A
+1158,3,3,JP,N/A
+1159,5,1,IT,N/A
+1160,3,4,CN,N/A
+1161,2,5,IT,N/A
+1162,4,5,CN,N/A
+1163,3,5,FR,N/A
+1164,1,2,FR,N/A
+1165,0,3,GB,N/A
+1166,5,4,CN,N/A
+1167,4,5,IT,N/A
+1168,3,2,IT,N/A
+1169,3,2,DE,N/A
+1170,2,2,DE,N/A
+1171,3,1,IT,N/A
+1172,3,1,IT,N/A
+1173,3,5,CN,N/A
+1174,5,2,US,N/A
+1175,2,3,FR,N/A
+1176,5,5,JP,N/A
+1177,1,4,DE,N/A
+1178,4,5,GB,N/A
+1179,5,3,FR,N/A
+1180,2,3,GB,N/A
+1181,0,3,IT,N/A
+1182,3,2,JP,N/A
+1183,5,5,IT,N/A
+1184,4,1,IT,N/A
+1185,3,2,IT,N/A
+1186,3,2,CN,N/A
+1187,2,5,IT,N/A
+1188,3,2,JP,N/A
+1189,4,4,IT,N/A
+1190,4,3,GB,N/A
+1191,3,4,JP,N/A
+1192,2,2,FR,N/A
+1193,5,1,FR,N/A
+1194,5,2,US,N/A
+1195,2,5,FR,N/A
+1196,1,2,FR,N/A
+1197,5,3,FR,N/A
+1198,1,4,GB,N/A
+1199,1,3,FR,N/A
+1200,4,5,IT,N/A
+1201,3,1,GB,N/A
+1202,0,3,GB,N/A
+1203,4,3,DE,N/A
+1204,4,2,GB,N/A
+1205,4,2,DE,N/A
+1206,5,2,GB,N/A
+1207,1,5,DE,N/A
+1208,4,5,GB,N/A
+1209,0,3,FR,N/A
+1210,1,5,FR,N/A
+1211,3,2,DE,N/A
+1212,5,1,JP,N/A
+1213,0,4,CN,N/A
+1214,3,1,JP,N/A
+1215,5,1,CN,N/A
+1216,4,2,FR,N/A

[39/50] [abbrv] kylin git commit: KYLIN-2245 code review

2016-12-06 Thread lidong

KYLIN-2245 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fc1e11aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fc1e11aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fc1e11aa

Branch: refs/heads/master-hbase1.x
Commit: fc1e11aa8a2d20add72e382396787d9cb09771da
Parents: 0a441c3
Author: Yang Li 
Authored: Mon Dec 5 21:20:21 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 5 21:20:21 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeInstance.java| 4 
 .../src/main/java/org/apache/kylin/cube/CubeManager.java | 2 +-
 .../main/java/org/apache/kylin/metadata/model/ISegment.java  | 4 ++--
 .../main/java/org/apache/kylin/metadata/model/Segments.java  | 8 
 4 files changed, 7 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index ecbb437..a3665f5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -371,10 +371,6 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return segments.calculateToBeSegments(newSegment, 
getModel().getPartitionDesc().isPartitioned());
 }
 
-public Pair 
findMergeOffsetsByDateRange(Segments segs, long startDate, long 
endDate, long skipSegDateRangeCap) {
-return this.segments.findMergeOffsetsByDateRange(segs, startDate, 
endDate, skipSegDateRangeCap);
-}
-
 public CubeSegment getLastSegment() {
 List existing = getSegments();
 if (existing.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 296a4e7..0c80c07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -519,7 +519,7 @@ public class CubeManager implements IRealizationProvider {
 if (isOffsetsOn) {
 // offset cube, merge by date range?
 if (startOffset == endOffset) {
-Pair pair = 
cube.findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), 
startDate, endDate, Long.MAX_VALUE);
+Pair pair = 
cube.getSegments(SegmentStatusEnum.READY).findMergeOffsetsByDateRange(startDate,
 endDate, Long.MAX_VALUE);
 if (pair == null)
 throw new IllegalArgumentException("Find no segments to 
merge by date range " + startDate + "-" + endDate + " for cube " + cube);
 startOffset = pair.getFirst().getSourceOffsetStart();

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index 9d26927..f006613 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -26,6 +26,8 @@ public interface ISegment {
 
 public long getDateRangeEnd();
 
+public boolean isSourceOffsetsOn();
+
 public long getSourceOffsetStart();
 
 public long getSourceOffsetEnd();
@@ -35,6 +37,4 @@ public interface ISegment {
 public SegmentStatusEnum getStatus();
 
 public long getLastBuildTime();
-
-public boolean isSourceOffsetsOn();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index bc115cc..5198dc7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -187,8 +187,8 @@ public class Segments extends 
ArrayList {

[33/50] [abbrv] kylin git commit: minor, change isReverse visibility in BuiltInFunctionTupleFilter.java

2016-12-06 Thread lidong

minor, change isReverse visibility in BuiltInFunctionTupleFilter.java


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00a4c506
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00a4c506
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00a4c506

Branch: refs/heads/master-hbase1.x
Commit: 00a4c5064bfe894bb8c106a214277d4d1d0f3c70
Parents: a03eec8
Author: Hongbin Ma 
Authored: Sun Dec 4 21:41:07 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Dec 4 21:41:07 2016 +0800

--
 .../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/00a4c506/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index f6e687b..b678394 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -47,7 +47,7 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 protected Method method;
 protected List methodParams;
 protected boolean isValidFunc = false;
-private boolean isReversed = false;
+protected boolean isReversed = false;
 
 public BuiltInFunctionTupleFilter(String name) {
 this(name, null);

[09/50] [abbrv] kylin git commit: KYLIN-1875 Add JoinsTree, refactor ModelChooser

2016-12-06 Thread lidong

KYLIN-1875 Add JoinsTree, refactor ModelChooser


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bda8787d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bda8787d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bda8787d

Branch: refs/heads/master-hbase1.x
Commit: bda8787d1a5c9bbe7d94798091763e1004a1eb61
Parents: a1a2a4a
Author: Yang Li 
Authored: Mon Nov 28 07:49:11 2016 +0800
Committer: Yang Li 
Committed: Tue Nov 29 07:25:29 2016 +0800

--
 .../apache/kylin/dict/DictionaryManager.java|   6 +-
 .../kylin/metadata/model/DataModelDesc.java |  30 +-
 .../apache/kylin/metadata/model/JoinDesc.java   |  11 +
 .../apache/kylin/metadata/model/JoinsTree.java  | 130 ++
 .../apache/kylin/metadata/model/TblColRef.java  |   4 +
 .../kylin/metadata/MetadataManagerTest.java |  11 +-
 .../kylin/metadata/model/JoinsTreeTest.java |  74 
 .../test_kylin_snowflake_sales_cube.json| 268 
 .../test_kylin_snowflake_model_desc.json| 161 
 .../test_kylin_snowflake_sales_model.json   |  87 
 .../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json |  28 ++
 .../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json  | 408 +++
 .../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json  | 152 +++
 .../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json |  24 ++
 .../localmeta/table/SNOWTEST.KYLIN_SALES.json   |  56 +++
 .../apache/kylin/query/relnode/OLAPContext.java |   2 +
 .../kylin/query/relnode/OLAPTableScan.java  |   7 +-
 .../kylin/query/routing/ModelChooser.java   |  55 +--
 18 files changed, 1293 insertions(+), 221 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 820299c..6178234 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -339,7 +339,7 @@ public class DictionaryManager {
 
 // find a lookup table that the col joins as FK
 for (TableRef lookup : model.getLookupTables()) {
-JoinDesc lookupJoin = model.getPKSideJoinMap().get(lookup);
+JoinDesc lookupJoin = model.getJoinByPKSide(lookup);
 int find = ArrayUtils.indexOf(lookupJoin.getForeignKeyColumns(), 
col);
 if (find < 0)
 continue;
@@ -357,8 +357,8 @@ public class DictionaryManager {
 if (join.isInnerJoin() == false)
 return false;
 
-TableRef table = join.getForeignKeyColumns()[0].getTableRef();
-join = model.getPKSideJoinMap().get(table);
+TableRef table = join.getFKSide();
+join = model.getJoinByPKSide(table);
 }
 return true;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index ff92def..d917571 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -97,8 +97,7 @@ public class DataModelDesc extends RootPersistentEntity {
 private Set allTableRefs = Sets.newLinkedHashSet();
 private Map aliasMap = Maps.newHashMap(); // alias => 
TableRef, a table has exactly one alias
 private Map tableNameMap = Maps.newHashMap(); // name => 
TableRef, a table maybe referenced by multiple names
-private Map pkSideJoinMap = Maps.newHashMap(); // 
table (PK side) => JoinTable
-private Map fkSideJoinMap = Maps.newHashMap(); // 
table (FK side) => JoinDesc
+private JoinsTree joinsTree;
 
 /**
  * Error messages during resolving json metadata
@@ -151,12 +150,12 @@ public class DataModelDesc extends RootPersistentEntity {
 return joinTables;
 }
 
-public Map getPKSideJoinMap() {
-return pkSideJoinMap;
+public JoinDesc getJoinByPKSide(TableRef table) {
+return joinsTree.getJoinByPKSide(table);
 }
-
-public Map getFKSideJoinMap() {
-return fkSideJoinMap;
+
+public JoinsTree

[35/50] [abbrv] kylin git commit: KYLIN-2236 let query cache honor backdoortoggles in query request

2016-12-06 Thread lidong

KYLIN-2236 let query cache honor backdoortoggles in query request


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8f3239bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8f3239bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8f3239bf

Branch: refs/heads/master-hbase1.x
Commit: 8f3239bf9ea4f1abd8d2c814967d425d4da68a45
Parents: 9556152
Author: Hongbin Ma 
Authored: Mon Dec 5 13:43:03 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 5 13:43:03 2016 +0800

--
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 30 ++
 .../apache/kylin/rest/request/SQLRequest.java   | 63 +++-
 .../apache/kylin/rest/service/QueryService.java |  1 +
 3 files changed, 53 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3239bf/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index 4c847bf..2f8991b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -261,6 +261,36 @@ public class ITJDBCDriverTest extends 
HBaseMetadataTestCase {
 
 }
 
+
+@Test
+public void testResultSetWithMaxRows() throws Exception {
+String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as 
TRANS_CNT from test_kylin_fact \n" + " group by LSTG_FORMAT_NAME ";
+
+Connection conn = getConnection();
+Statement statement = conn.createStatement();
+statement.setMaxRows(2);
+
+statement.execute(sql);
+
+ResultSet rs = statement.getResultSet();
+
+int count = 0;
+while (rs.next()) {
+count++;
+String lstg = rs.getString(1);
+double gmv = rs.getDouble(2);
+int trans_count = rs.getInt(3);
+
+System.out.println("Get a line: LSTG_FORMAT_NAME=" + lstg + ", 
GMV=" + gmv + ", TRANS_CNT=" + trans_count);
+}
+
+Assert.assertTrue(count == 2);
+statement.close();
+rs.close();
+conn.close();
+
+}
+
 private static class SystemPropertiesOverride {
 HashMap backup = new HashMap();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3239bf/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java 
b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index 96f5faa..bd8b7e2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -83,50 +83,31 @@ public class SQLRequest implements Serializable {
 this.acceptPartial = acceptPartial;
 }
 
+
 @Override
-public int hashCode() {
-final int prime = 31;
-int result = 1;
-result = prime * result + (acceptPartial ? 1231 : 1237);
-result = prime * result + ((offset == null) ? 0 : offset.hashCode());
-result = prime * result + ((limit == null) ? 0 : limit.hashCode());
-result = prime * result + ((project == null) ? 0 : project.hashCode());
-result = prime * result + ((sql == null) ? 0 : sql.hashCode());
-return result;
+public boolean equals(Object o) {
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
+
+SQLRequest that = (SQLRequest) o;
+
+if (acceptPartial != that.acceptPartial) return false;
+if (sql != null ? !sql.equals(that.sql) : that.sql != null) return 
false;
+if (project != null ? !project.equals(that.project) : that.project != 
null) return false;
+if (offset != null ? !offset.equals(that.offset) : that.offset != 
null) return false;
+if (limit != null ? !limit.equals(that.limit) : that.limit != null) 
return false;
+return backdoorToggles != null ? 
backdoorToggles.equals(that.backdoorToggles) : that.backdoorToggles == null;
+
 }
 
 @Override
-public boolean equals(Object obj) {
-if (this == obj)
-return true;
-if (obj == null)
-return false;
-if (getClass() != obj.getClass())
-return false;
-SQLRequest other = (SQLRequest) obj;
-if (acceptPartial != other.acceptPartial)
-return false;
-if (offset == null) {
-if

[05/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'

2016-12-06 Thread lidong

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index eb17922..48eef46 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -6,27 +6,32 @@
   "description" : null,
   "null_string" : null,
   "dimensions" : [ {
+"name" : "TRANS_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "TRANS_ID",
+"derived" : null
+  }, {
 "name" : "CAL_DT",
 "table" : "DEFAULT.KYLIN_CAL_DT",
 "column" : "{FK}",
-"derived" : [ "WEEK_BEG_DT" ]
+"derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
   }, {
 "name" : "CATEGORY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "{FK}",
 "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
   }, {
-"name" : "CATEGORY_HIERARCHY",
+"name" : "META_CATEG_NAME",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "META_CATEG_NAME",
 "derived" : null
   }, {
-"name" : "CATEGORY_HIERARCHY",
+"name" : "CATEG_LVL2_NAME",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "CATEG_LVL2_NAME",
 "derived" : null
   }, {
-"name" : "CATEGORY_HIERARCHY",
+"name" : "CATEG_LVL3_NAME",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "CATEG_LVL3_NAME",
 "derived" : null
@@ -36,14 +41,54 @@
 "column" : "LSTG_FORMAT_NAME",
 "derived" : null
   }, {
-"name" : "USER_ID",
+"name" : "SELLER_ID",
 "table" : "DEFAULT.KYLIN_SALES",
-"column" : "USER_ID",
+"column" : "SELLER_ID",
 "derived" : null
   }, {
-"name" : "REGION",
+"name" : "BUYER_ID",
 "table" : "DEFAULT.KYLIN_SALES",
-"column" : "REGION",
+"column" : "BUYER_ID",
+"derived" : null
+  }, {
+"name" : "ACCOUNT_BUYER_LEVEL",
+"table" : "BUYER_ACCOUNT",
+"column" : "ACCOUNT_BUYER_LEVEL",
+"derived" : null
+  }, {
+"name" : "ACCOUNT_SELLER_LEVEL",
+"table" : "SELLER_ACCOUNT",
+"column" : "ACCOUNT_SELLER_LEVEL",
+"derived" : null
+  }, {
+"name" : "BUYER_COUNTRY",
+"table" : "BUYER_ACCOUNT",
+"column" : "ACCOUNT_COUNTRY",
+"derived" : null
+  }, {
+"name" : "SELLER_COUNTRY",
+"table" : "SELLER_ACCOUNT",
+"column" : "ACCOUNT_COUNTRY",
+"derived" : null
+  }, {
+"name" : "BUYER_COUNTRY_NAME",
+"table" : "BUYER_COUNTRY",
+"column" : "NAME",
+"derived" : null
+  }, {
+"name" : "SELLER_COUNTRY_NAME",
+"table" : "SELLER_COUNTRY",
+"column" : "NAME",
+"derived" : null
+  }, {
+"name" : "OPS_USER_ID",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "OPS_USER_ID",
+"derived" : null
+  }, {
+"name" : "OPS_REGION",
+"table" : "DEFAULT.KYLIN_SALES",
+"column" : "OPS_REGION",
 "derived" : null
   } ],
   "measures" : [ {
@@ -56,8 +101,7 @@
 "next_parameter" : null
   },
   "returntype" : "decimal(19,4)"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "GMV_MIN",
 "function" : {
@@ -68,8 +112,7 @@
 "next_parameter" : null
   },
   "returntype" : "decimal(19,4)"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "GMV_MAX",
 "function" : {
@@ -80,8 +123,7 @@
 "next_parameter" : null
   },
   "returntype" : "decimal(19,4)"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "TRANS_CNT",
 "function" : {
@@ -92,8 +134,7 @@
 "next_parameter" : null
   },
   "returntype" : "bigint"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "SELLER_CNT_HLL",
 "function" : {
@@ -104,20 +145,7 @@
 "next_parameter" : null
   },
   "returntype" : "hllc(10)"
-},
-"dependent_measure_ref" : null
-  }, {
-"name" : "SELLER_FORMAT_CNT",
-"function" : {
-  "expression" : "COUNT_DISTINCT",
-  "parameter" : {
-"type" : "column",
-"value" : "LSTG_FORMAT_NAME",
-"next_parameter" : null
-  },
-  "returntype" : "hllc(10)"
-},
-"dependent_measure_ref" : null
+}
   }, {
 "name" : "TOP_SELLER",
 "function" : {
@@ -132,13 +160,21 @@
 }
   },
   "returntype" : "topn(100)"
-},
-"dependent_measure_ref" : null
+}
   } ],
   "rowkey" : {
 "rowkey_columns" : [ {
+  "column" : "BUYER_ID",
+  "encoding" : "integer:4"
+}, {
+  "column" : "SELLER_ID",
+  "encoding" : "integer:4"
+}, {
+  "column" : "TRANS_ID",
+  "encoding" : "integer:4"
+}, {
   "column" : "PART_DT",
-  "encoding" : "dict"
+  "encoding" : "date"
 }, {
   "column" : "LEAF_CATEG_ID",
   "encoding" :

[50/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-12-06 Thread lidong

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/469d9cc5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/469d9cc5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/469d9cc5

Branch: refs/heads/master-hbase1.x
Commit: 469d9cc5c7df0a648d652635a7dc6b9ef4383182
Parents: 081ed0d
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Wed Dec 7 12:28:13 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[27/50] [abbrv] kylin git commit: KYLIN-2245 refine CubeSegments

2016-12-06 Thread lidong

KYLIN-2245 refine CubeSegments

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ca53954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ca53954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ca53954

Branch: refs/heads/master-hbase1.x
Commit: 3ca53954415c62c2a91473c6d7bf831504b5e9db
Parents: 4a0ee79
Author: Cheng Wang 
Authored: Fri Dec 2 15:35:52 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 17:26:07 2016 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java | 115 +++
 .../java/org/apache/kylin/cube/CubeManager.java |  10 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   8 +-
 .../apache/kylin/metadata/model/ISegment.java   |   6 +-
 .../apache/kylin/metadata/model/Segments.java   | 140 +++
 .../hbase/util/ExtendCubeToHybridCLI.java   |   4 +-
 .../kylin/tool/ExtendCubeToHybridCLI.java   |   4 +-
 7 files changed, 176 insertions(+), 111 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 061ab23..8b12c2e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,8 +18,6 @@
 
 package org.apache.kylin.cube;
 
-import java.util.ArrayList;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
@@ -34,6 +32,7 @@ import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.realization.CapabilityResult;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
@@ -64,7 +63,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 cubeInstance.setName(cubeName);
 cubeInstance.setDescName(cubeDesc.getName());
 cubeInstance.setCreateTimeUTC(System.currentTimeMillis());
-cubeInstance.setSegments(new ArrayList());
+cubeInstance.setSegments(new Segments());
 cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
 cubeInstance.updateRandomUuid();
 
@@ -87,7 +86,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 
 @JsonManagedReference
 @JsonProperty("segments")
-private List segments = new ArrayList();
+private Segments segments = new Segments();
 
 @JsonProperty("create_time_utc")
 private long createTimeUTC;
@@ -97,39 +96,11 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 }
 
 public List getBuildingSegments() {
-List buildingSegments = new ArrayList();
-if (null != segments) {
-for (CubeSegment segment : segments) {
-if (SegmentStatusEnum.NEW == segment.getStatus() || 
SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
-buildingSegments.add(segment);
-}
-}
-}
-
-return buildingSegments;
+return segments.getBuildingSegments();
 }
 
 public List getMergingSegments(CubeSegment mergedSegment) {
-LinkedList result = new LinkedList();
-if (mergedSegment == null)
-return result;
-
-for (CubeSegment seg : this.segments) {
-if (seg.getStatus() != SegmentStatusEnum.READY && seg.getStatus() 
!= SegmentStatusEnum.READY_PENDING)
-continue;
-
-if (seg == mergedSegment)
-continue;
-
-if (mergedSegment.sourceOffsetContains(seg)) {
-// make sure no holes
-if (result.size() > 0 && result.getLast().getSourceOffsetEnd() 
!= seg.getSourceOffsetStart())
-throw new IllegalStateException("Merging segments must not 
have holes between " + result.getLast() + " and " + seg);
-
-result.add(seg);
-}
-}
-return result;
+return segments.getMergingSegments(mergedSegment);
 }
 
 public CubeDesc getDescriptor() {
@@ -154,7 +125,7 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 // in a temporary broken state, so that user can edit and fix it. Broken 
state is often due to
 //

[47/50] [abbrv] kylin git commit: KYLIN-2252, Enhance project/model/cube name check

2016-12-06 Thread lidong

KYLIN-2252, Enhance project/model/cube name check


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e787165
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e787165
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e787165

Branch: refs/heads/master-hbase1.x
Commit: 1e787165edee713a59db8627e70f79edabfd7d9d
Parents: ea13af4
Author: Billy Liu 
Authored: Wed Dec 7 09:42:58 2016 +0800
Committer: Billy Liu 
Committed: Wed Dec 7 09:42:58 2016 +0800

--
 .../org/apache/kylin/rest/controller/CubeController.java  | 10 ++
 .../apache/kylin/rest/controller/CubeDescController.java  |  4 ++--
 .../org/apache/kylin/rest/controller/ModelController.java | 10 ++
 .../apache/kylin/rest/controller/ProjectController.java   | 10 +-
 4 files changed, 31 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 3846d28..f537231 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -83,6 +83,8 @@ import com.google.common.collect.Maps;
 public class CubeController extends BasicController {
 private static final Logger logger = 
LoggerFactory.getLogger(CubeController.class);
 
+private static final char[] VALID_CUBENAME = 
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
+
 @Autowired
 private CubeService cubeService;
 
@@ -343,6 +345,10 @@ public class CubeController extends BasicController {
 if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
 throw new BadRequestException("Broken cube can't be cloned");
 }
+if (!StringUtils.containsOnly(newCubeName, VALID_CUBENAME)) {
+logger.info("Invalid Cube name {}, only letters, numbers and 
underline supported.", newCubeName);
+throw new BadRequestException("Invalid Cube name, only letters, 
numbers and underline supported.");
+}
 
 CubeDesc cubeDesc = cube.getDescriptor();
 CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
@@ -421,6 +427,10 @@ public class CubeController extends BasicController {
 logger.info("Cube name should not be empty.");
 throw new BadRequestException("Cube name should not be empty.");
 }
+if (!StringUtils.containsOnly(name, VALID_CUBENAME)) {
+logger.info("Invalid Cube name {}, only letters, numbers and 
underline supported.", name);
+throw new BadRequestException("Invalid Cube name, only letters, 
numbers and underline supported.");
+}
 
 try {
 desc.setUuid(UUID.randomUUID().toString());

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
index 5a8eeec..0c8f487 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -44,8 +44,8 @@ public class CubeDescController extends BasicController {
 /**
  * Get detail information of the "Cube ID"
  * 
- * @param cubeDescName
- *Cube ID
+ * @param cubeName
+ *Cube Name
  * @return
  * @throws IOException
  */

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
index 5f6a91b..df9ecfb 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -61,6 +61,8 @@ import com.fasterxml.jackson.databind.JsonMappingException;
 public class ModelController extends BasicController {
 private static final Logger logger =

[20/50] [abbrv] kylin git commit: KYLIN-2180 minor, get owenr project in CubeDesc.init()

2016-12-06 Thread lidong

KYLIN-2180 minor, get owenr project in CubeDesc.init()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9fc8f5e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9fc8f5e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9fc8f5e2

Branch: refs/heads/master-hbase1.x
Commit: 9fc8f5e207b5111fffaec3335efe0c8947353cfa
Parents: 1a5295b
Author: Li Yang 
Authored: Fri Dec 2 11:39:34 2016 +0800
Committer: Li Yang 
Committed: Fri Dec 2 11:39:34 2016 +0800

--
 .../main/java/org/apache/kylin/cube/model/CubeDesc.java  | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9fc8f5e2/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 327ce57..853571c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -64,6 +64,9 @@ import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -519,11 +522,16 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 public void init(KylinConfig config) {
 this.errors.clear();
-this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
 
 checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank");
 checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc(%s) has 
blank modelName", name);
 
+// note CubeDesc.name == CubeInstance.name
+List ownerPrj = 
ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name);
+logger.info("CubeDesc '" + name + "' is owned by " + ownerPrj);
+
+this.config = KylinConfigExt.createInstance(config, 
overrideKylinProps);
+
 this.model = 
MetadataManager.getInstance(config).getDataModelDesc(modelName);
 checkNotNull(this.model, "DateModelDesc(%s) not found", modelName);
 
@@ -823,6 +831,7 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 return col;
 }
 
+@SuppressWarnings("deprecation")
 private void initMeasureColumns() {
 if (measures == null || measures.isEmpty()) {
 return;

[19/50] [abbrv] kylin git commit: measure encoding length bug

2016-12-06 Thread lidong

measure encoding length bug

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a5295ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a5295ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a5295ba

Branch: refs/heads/master-hbase1.x
Commit: 1a5295bad21fc18766bb3c7ba6eee59678d42185
Parents: 5822492
Author: luguosheng <550175...@qq.com>
Authored: Thu Dec 1 17:54:35 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 18:17:35 2016 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 4 ++--
 webapp/app/js/controllers/cubeEdit.js   | 2 +-
 webapp/app/js/controllers/cubeMeasures.js   | 8 
 3 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index f2b36fe..8192419 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -47,7 +47,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var _encoding = item.encoding;
 var _valueLength ;
 var baseKey=item.encoding.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>=-1){
+if(needLengthKeyList.indexOf(baseKey)!=-1){
   var result=/:(\d+)/.exec(item.encoding);
   _valueLength=result?result[1]:0;
 }
@@ -78,7 +78,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 var version=$scope.getTypeVersion(item.encoding);
 var encodingType=$scope.removeVersion(item.encoding);
 
-if(needLengthKeyList.indexOf(encodingType)>=-1){
+if(needLengthKeyList.indexOf(encodingType)!=-1){
   encoding = encodingType+":"+item.valueLength;
 }else{
   encoding = encodingType;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 0704a62..85bd4b1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -78,7 +78,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 $scope.store.supportedEncoding = $scope.cubeConfig.encodings;
   })
   $scope.createFilter=function(type){
- if(type.indexOf("varchar")<=0){
+ if(type.indexOf("varchar")==-1){
return ['fixed_length_hex'];
  }else if(type!="date"){
return ['date'];

http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 085338f..18f53c2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -45,7 +45,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 }
   }
   $scope.createFilter=function(type){
-if(type.indexOf("varchar")<=0){
+if(type.indexOf("varchar")==-1){
   return ['fixed_length_hex'];
 }else if(type!="date"){
   return ['date'];
@@ -60,7 +60,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var encodings =$scope.store.supportedEncoding,filterEncoding=[];
 var filerList=$scope.createFilter(type);
 if($scope.isEdit) {
-  if (name && $scope.newMeasure.function.configuration) {
+  if (name && 
$scope.newMeasure.function.configuration&&$scope.newMeasure.function.configuration['topn.encoding.'
 + name]) {
 var version = 
$scope.newMeasure.function.configuration['topn.encoding_version.' + name] || 1;
 filterEncoding = VdmUtil.getFilterObjectListByOrFilterVal(encodings, 
'value', $scope.newMeasure.function.configuration['topn.encoding.' + 
name].replace(/:\d+/, "") + (version ? "[v" + version + "]" : "[v1]"), 
'suggest', true);
   }else{
@@ -119,7 +119,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 var 
version=$scope.newMeasure.function.configuration['topn.encoding_version.'+_name]||1;
 item=$scope.removeVersion(item);
 var baseKey=item.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>=-1){
+if(needLengthKeyList.indexOf(baseKey)!=-1){
   var result=/:(\d+)/.exec(item);

[01/50] [abbrv] kylin git commit: KYLIN-2213 UnsupportedOperationException when excute 'not like' query on cube v1 [Forced Update!]

2016-12-06 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/master-hbase1.x 36f930863 -> 469d9cc5c (forced update)


KYLIN-2213 UnsupportedOperationException when excute 'not like' query on cube v1

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be7b5d10
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be7b5d10
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be7b5d10

Branch: refs/heads/master-hbase1.x
Commit: be7b5d1012e4ba0f8696ccbf9c0507706c07ad22
Parents: 3f10b3a
Author: zhengdong 
Authored: Wed Nov 23 16:03:16 2016 +0800
Committer: Li Yang 
Committed: Mon Nov 28 16:23:38 2016 +0800

--
 .../apache/kylin/dict/BuiltInFunctionTransformer.java | 14 --
 .../metadata/filter/BuiltInFunctionTupleFilter.java   | 11 +++
 2 files changed, 23 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/be7b5d10/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
old mode 100644
new mode 100755
index 9707f8f..0f8e895
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -89,7 +89,12 @@ public class BuiltInFunctionTransformer implements 
ITupleFilterTransformer {
 if (dict == null)
 return null;
 
-CompareTupleFilter translated = new 
CompareTupleFilter(FilterOperatorEnum.IN);
+CompareTupleFilter translated;
+if (builtInFunctionTupleFilter.isReversed()) {
+translated = new CompareTupleFilter(FilterOperatorEnum.NOTIN);
+} else {
+translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+}
 translated.addChild(new ColumnTupleFilter(columnRef));
 
 try {
@@ -121,7 +126,12 @@ public class BuiltInFunctionTransformer implements 
ITupleFilterTransformer {
 if (dict == null)
 return null;
 
-CompareTupleFilter translated = new 
CompareTupleFilter(FilterOperatorEnum.IN);
+CompareTupleFilter translated;
+if (builtInFunctionTupleFilter.isReversed()) {
+translated = new CompareTupleFilter(FilterOperatorEnum.NOTIN);
+} else {
+translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+}
 translated.addChild(new ColumnTupleFilter(columnRef));
 
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/be7b5d10/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
old mode 100644
new mode 100755
index 734b374..1e836b7
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -47,6 +47,7 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 protected Method method;
 protected List methodParams;
 protected boolean isValidFunc = false;
+private boolean isReversed = false;
 
 public BuiltInFunctionTupleFilter(String name) {
 this(name, null);
@@ -99,6 +100,16 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 }
 
 @Override
+public TupleFilter reverse() {
+isReversed = !isReversed;
+return this;
+}
+
+public boolean isReversed() {
+return isReversed;
+}
+
+@Override
 public void addChild(TupleFilter child) {
 if (child instanceof ColumnTupleFilter || child instanceof 
BuiltInFunctionTupleFilter) {
 columnContainerFilter = child;

[08/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'

2016-12-06 Thread lidong

KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 
'kylin_default_instance'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1a2a4a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1a2a4a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1a2a4a6

Branch: refs/heads/master-hbase1.x
Commit: a1a2a4a643d013471079ac0d353bbf179fd0d85c
Parents: c3d62e9
Author: Yang Li 
Authored: Sun Nov 27 18:26:33 2016 +0800
Committer: Yang Li 
Committed: Tue Nov 29 07:25:25 2016 +0800

--
 build/conf/kylin.properties | 2 +-
 .../apache/kylin/common/KylinConfigBase.java|11 +-
 .../kylin/metadata/model/DataModelDesc.java | 3 +
 .../kylin/metadata/model/MeasureDesc.java   | 5 +
 .../realization/IRealizationConstants.java  | 2 +-
 examples/sample_cube/create_sample_tables.sql   |21 +-
 .../sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv  | 1 +
 .../sample_cube/data/DEFAULT.KYLIN_SALES.csv| 2 -
 .../template/cube_desc/kylin_sales_cube.json|   157 +-
 .../template/model_desc/kylin_sales_model.json  |50 +-
 .../template/table/DEFAULT.KYLIN_ACCOUNT.json   |28 +
 .../template/table/DEFAULT.KYLIN_SALES.json |12 +-
 .../test_case_data/sandbox/kylin.properties | 2 +-
 .../apache/kylin/query/relnode/OLAPJoinRel.java |25 +-
 .../kylin/rest/security/AclHBaseStorage.java| 1 -
 .../rest/security/RealAclHBaseStorage.java  | 5 +-
 .../apache/kylin/rest/service/QueryService.java | 6 +-
 .../kylin/storage/hbase/HBaseResourceStore.java | 2 +-
 .../storage/hbase/util/StorageCleanupJob.java   | 2 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 2 +-
 20 files changed, 20246 insertions(+), 10090 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 1409394..131a725 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -18,7 +18,7 @@
 ### METADATA | ENV ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
+kylin.metadata.url=kylin_default_instance@hbase
 
 # Working folder in HDFS, make sure user has the right access to the hdfs 
directory
 kylin.env.hdfs-working-dir=/kylin

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index aa5e6e1..7dcc771 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -188,13 +188,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public String getMetadataUrlPrefix() {
-String hbaseMetadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_metadata";
+String metadataUrl = getMetadataUrl();
+String defaultPrefix = "kylin_default_instance";
 
-if 
(org.apache.commons.lang3.StringUtils.containsIgnoreCase(hbaseMetadataUrl, 
"@hbase")) {
-int cut = hbaseMetadataUrl.indexOf('@');
-String tmp = cut < 0 ? defaultPrefix : 
hbaseMetadataUrl.substring(0, cut);
-return tmp;
+if (metadataUrl.endsWith("@hbase")) {
+int cut = metadataUrl.lastIndexOf('@');
+return metadataUrl.substring(0, cut);
 } else {
 return defaultPrefix;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index c2de5d6..ff92def 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -378,6 +378,9 @@ public class DataModelDesc extends RootPersistentEntity {
 for (int i = 0; i < pks.length; i++) {
 TblColRef col = dimTable.getColumn(pks[i]);
 if (col == null) {
+col = findColumn(pks[i]);
+}
+if (col == null || col.getTableRef().equals(dimTable) == 
false)

[12/50] [abbrv] kylin git commit: filter null value in cubedesc and model desc

2016-12-06 Thread lidong

filter null value in cubedesc and model desc

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/875a5931
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/875a5931
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/875a5931

Branch: refs/heads/master-hbase1.x
Commit: 875a5931ad8cad79cdec1e80ab9388ded9446091
Parents: 3186d17
Author: luguosheng <550175...@qq.com>
Authored: Wed Nov 30 10:51:51 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 30 13:48:29 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js  |  4 ++--
 webapp/app/js/controllers/modelEdit.js | 13 ++---
 webapp/app/js/utils/utils.js   | 24 
 3 files changed, 36 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 888662d..0704a62 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -413,7 +413,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 
 if ($scope.isEdit) {
   CubeService.update({}, {
-cubeDescData: $scope.state.cubeSchema,
+cubeDescData: VdmUtil.filterNullValInObj($scope.state.cubeSchema),
 cubeName: $routeParams.cubeName,
 project: $scope.state.project
   }, function (request) {
@@ -453,7 +453,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   });
 } else {
   CubeService.save({}, {
-cubeDescData: $scope.state.cubeSchema,
+cubeDescData: VdmUtil.filterNullValInObj($scope.state.cubeSchema),
 project: $scope.state.project
   }, function (request) {
 if (request.successful) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/controllers/modelEdit.js
--
diff --git a/webapp/app/js/controllers/modelEdit.js 
b/webapp/app/js/controllers/modelEdit.js
index 0b3db66..2f1b35e 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -19,7 +19,7 @@
 'use strict';
 
 
-KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, 
$location, $templateCache, $interpolate, MessageService, TableService, 
CubeDescService, ModelService, loadingRequest, 
SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager)
 {
+KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, 
$location, $templateCache, $interpolate, MessageService, TableService, 
CubeDescService, ModelService, loadingRequest, 
SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager,VdmUtil)
 {
 //add or edit ?
 var absUrl = $location.absUrl();
 $scope.modelMode = 
absUrl.indexOf("/models/add")!=-1?'addNewModel':absUrl.indexOf("/models/edit")!=-1?'editExistModel':'default';
@@ -188,7 +188,11 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, 
$routeParams, $locati
 loadingRequest.show();
 
 if ($scope.isEdit) {
-ModelService.update({}, 
{modelDescData:$scope.state.modelSchema, modelName: $routeParams.modelName, 
project: $scope.state.project}, function (request) {
+ModelService.update({}, {
+  
modelDescData:VdmUtil.filterNullValInObj($scope.state.modelSchema),
+  modelName: $routeParams.modelName,
+  project: $scope.state.project
+}, function (request) {
 if (request.successful) {
 $scope.state.modelSchema = request.modelSchema;
 SweetAlert.swal('', 'Updated the model 
successfully.', 'success');
@@ -216,7 +220,10 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, 
$routeParams, $locati
 loadingRequest.hide();
 });
 } else {
-ModelService.save({}, 
{modelDescData:$scope.state.modelSchema, project: $scope.state.project}, 
function (request) {
+ModelService.save({}, {
+  
modelDescData:VdmUtil.filterNullValInObj($scope.state.modelSchema),
+  project: $scope.state.project
+}, function (request) {
 if(request.successful) {

[36/50] [abbrv] kylin git commit: KYLIN-2212 add more test queries

2016-12-06 Thread lidong

KYLIN-2212 add more test queries


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af429e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af429e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af429e5c

Branch: refs/heads/master-hbase1.x
Commit: af429e5cbbda2a64159b25bc5ce4ad70f2f1a1f2
Parents: 8f3239b
Author: Hongbin Ma 
Authored: Mon Dec 5 15:05:35 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 5 15:05:35 2016 +0800

--
 .../metadata/filter/LogicalTupleFilter.java |  3 +-
 .../src/test/resources/query/sql/query101.sql   | 13 
 .../src/test/resources/query/sql/query102.sql   | 13 
 .../src/test/resources/query/sql/query103.sql   | 13 
 .../test/resources/query/sql_like/query21.sql   | 31 
 5 files changed, 71 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 61657fb..1744309 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -64,8 +64,7 @@ public class LogicalTupleFilter extends TupleFilter {
 public TupleFilter reverse() {
 switch (operator) {
 case NOT:
-throw new IllegalStateException("not( not in ()) is invalid 
syntax");
-//return reverseNestedNots(this, 0);
+throw new IllegalStateException("NOT will be replaced in 
org.apache.kylin.query.relnode.OLAPFilterRel.TupleFilterVisitor");
 case AND:
 case OR:
 LogicalTupleFilter reverse = new 
LogicalTupleFilter(REVERSE_OP_MAP.get(operator));

http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query101.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query101.sql 
b/kylin-it/src/test/resources/query/sql/query101.sql
new file mode 100644
index 000..fb42bca
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query101.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV 
+
+ from test_kylin_fact 
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') )
+ group by meta_categ_name 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query102.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query102.sql 
b/kylin-it/src/test/resources/query/sql/query102.sql
new file mode 100644
index 000..bd1e15e
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query102.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV 
+
+ from test_kylin_fact 
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') and meta_categ_name 
not in ('Crafts','Computers'))
+ group by meta_categ_name 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query103.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query103.sql 
b/kylin-it/src/test/resources/query/sql/query103.sql
new file mode 100644
index 000..c5f9bf9
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query103.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV 
+
+ from test_kylin_fact 
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON

[34/50] [abbrv] kylin git commit: minor, add setter for BuiltInFunctionTupleFilter's reversed field

2016-12-06 Thread lidong

minor, add setter for BuiltInFunctionTupleFilter's reversed field


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95561526
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95561526
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95561526

Branch: refs/heads/master-hbase1.x
Commit: 9556152622599a87610f16ae8f06185e24ec02d6
Parents: 00a4c50
Author: Hongbin Ma 
Authored: Mon Dec 5 10:58:31 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Dec 5 10:58:31 2016 +0800

--
 .../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/95561526/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index b678394..90123c4 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -109,6 +109,10 @@ public class BuiltInFunctionTupleFilter extends 
FunctionTupleFilter {
 return isReversed;
 }
 
+public void setReversed(boolean reversed) {
+this.isReversed = reversed;
+}
+
 @Override
 public void addChild(TupleFilter child) {
 if (child instanceof ColumnTupleFilter || child instanceof 
BuiltInFunctionTupleFilter) {

[37/50] [abbrv] kylin git commit: KYLIN-2248 TopN merge further optimization after KYLIN-1917

2016-12-06 Thread lidong

KYLIN-2248 TopN merge further optimization after KYLIN-1917


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59a30f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59a30f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59a30f66

Branch: refs/heads/master-hbase1.x
Commit: 59a30f66d47cc1838e6852405699fd7957bfac29
Parents: af429e5
Author: shaofengshi 
Authored: Sun Dec 4 09:39:45 2016 +0800
Committer: shaofengshi 
Committed: Mon Dec 5 17:42:34 2016 +0800

--
 .../apache/kylin/measure/topn/TopNCounter.java  | 47 ++--
 1 file changed, 13 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59a30f66/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 968e694..caf7961 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -26,11 +26,9 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import com.google.common.collect.Maps;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 
 /**
  * Modified from the StreamSummary.java in 
https://github.com/addthis/stream-lib
@@ -157,41 +155,22 @@ public class TopNCounter implements 
Iterable {
  * @return
  */
 public TopNCounter merge(TopNCounter another) {
-double m1 = 0.0, m2 = 0.0;
-if (this.size() >= this.capacity) {
-m1 = this.counterList.getLast().count;
-}
-
-if (another.size() >= another.capacity) {
-m2 = another.counterList.getLast().count;
-}
-
-Set duplicateItems = Sets.newHashSet();
-List notDuplicateItems = Lists.newArrayList();
-
-for (Map.Entry entry : this.counterMap.entrySet()) {
-T item = entry.getKey();
-Counter existing = another.counterMap.get(item);
-if (existing != null) {
-duplicateItems.add(item);
-} else {
-notDuplicateItems.add(item);
+boolean thisFull = this.size() >= this.capacity;
+boolean anotherFull = another.size() >= another.capacity;
+double m1 = thisFull ? this.counterList.getLast().count : 0.0;
+double m2 = anotherFull ? another.counterList.getLast().count : 0.0;
+
+if (anotherFull == true) {
+for (Counter entry : this.counterMap.values()) {
+entry.count += m2;
 }
 }
 
-for (T item : duplicateItems) {
-this.offer(item, another.counterMap.get(item).count);
-}
-
-for (T item : notDuplicateItems) {
-this.offer(item, m2);
-}
-
 for (Map.Entry entry : another.counterMap.entrySet()) {
-T item = entry.getKey();
-if (duplicateItems.contains(item) == false) {
-double counter = entry.getValue().count;
-this.offer(item, counter + m1);
+if (this.counterMap.containsKey(entry.getKey())) {
+this.offer(entry.getValue().getItem(), (entry.getValue().count 
- m2));
+} else {
+this.offer(entry.getValue().getItem(), entry.getValue().count 
+ m1);
 }
 }

[17/50] [abbrv] kylin git commit: KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally

2016-12-06 Thread lidong

 KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28ba1eae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28ba1eae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28ba1eae

Branch: refs/heads/master-hbase1.x
Commit: 28ba1eaeac8fd29ff3872f6d14604813a89b3a6e
Parents: 93bf0d0
Author: Hongbin Ma 
Authored: Thu Dec 1 11:35:34 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Dec 1 11:46:00 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java   | 12 
 .../main/java/org/apache/kylin/cube/model/CubeDesc.java |  6 +-
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28ba1eae/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 766c04d..3c10826 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -79,7 +79,7 @@ abstract public class KylinConfigBase implements Serializable 
{
 protected KylinConfigBase(Properties props, boolean force) {
 this.properties = force ? props : BCC.check(props);
 }
-
+
 final protected String getOptional(String prop) {
 return getOptional(prop, null);
 }
@@ -161,7 +161,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public boolean isDevEnv() {
 return "DEV".equals(getOptional("kylin.env", "DEV"));
 }
-
+
 public String getDeployEnv() {
 return getOptional("kylin.env", "DEV");
 }
@@ -211,7 +211,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public Map getCubeCustomMeasureTypes() {
 return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
 }
-
+
 // 

 // DICTIONARY & SNAPSHOT
 // 

@@ -278,6 +278,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
 }
 
+public boolean isIgnoreCubeSignatureInconsistency() {
+return 
Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", 
"false"));
+}
+
 @Deprecated
 public int getCubeAggrGroupMaxSize() {
 return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", 
"12"));
@@ -298,7 +302,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public void setMaxBuildingSegments(int maxBuildingSegments) {
 setProperty("kylin.cube.max-building-segments", 
String.valueOf(maxBuildingSegments));
 }
-
+
 // 

 // JOB
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/28ba1eae/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 47063d0..327ce57 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -466,6 +466,11 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
  * @return
  */
 public boolean checkSignature() {
+if (this.getConfig().isIgnoreCubeSignatureInconsistency()) {
+logger.info("Skip checking cube signature");
+return true;
+}
+
 if (KylinVersion.getCurrentVersion().isCompatibleWith(new 
KylinVersion(getVersion())) && 
!KylinVersion.getCurrentVersion().isSignatureCompatibleWith(new 
KylinVersion(getVersion( {
 logger.info("checkSignature on {} is skipped as the its version is 
{} (not signature compatible but compatible) ", getName(), getVersion());
 return true;
@@ -1102,7 +1107,6 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 return newCubeDesc;
 }
 
-
 private Collection ensureOrder(Collection c) {
 TreeSet set = new TreeSet();
 for (Object o : c)

[10/50] [abbrv] kylin git commit: KYLIN-2195 could run get-properties.sh in source code dir

2016-12-06 Thread lidong

KYLIN-2195 could run get-properties.sh in source code dir


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8506e4d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8506e4d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8506e4d4

Branch: refs/heads/master-hbase1.x
Commit: 8506e4d4f75fbbda518997cac97e229fa5a0
Parents: bda8787
Author: lidongsjtu 
Authored: Tue Nov 29 18:24:07 2016 +0800
Committer: lidongsjtu 
Committed: Tue Nov 29 18:24:07 2016 +0800

--
 build/bin/get-properties.sh | 12 ++--
 1 file changed, 10 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8506e4d4/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 170442d..0fa4f6f 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -19,13 +19,21 @@
 
 source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 
+
 if [ $# != 1 ]
 then
 echo 'invalid input'
 exit -1
 fi
 
-job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
-tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+if [[ $CI_MODE == 'true' ]]; then
+cd $dir
+job_jar=$(ls ../../assembly/target/kylin-*-job.jar)
+tool_jar=$(ls ../../tool/target/kylin-tool-*.jar|grep -v assembly)
+else
+job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+fi
+
 result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 
2>/dev/null`
 echo "$result"

[26/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global Dictionary

2016-12-06 Thread lidong

KYLIN-2192 More Robust Global Dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a0ee798
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a0ee798
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a0ee798

Branch: refs/heads/master-hbase1.x
Commit: 4a0ee7989d5f8272592b980fce3f5716ca40d4c1
Parents: e562aaf
Author: sunyerui 
Authored: Mon Nov 21 21:26:34 2016 +0800
Committer: gaodayue 
Committed: Fri Dec 2 13:33:59 2016 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   |  12 +-
 .../apache/kylin/common/KylinConfigBase.java|   8 +
 .../apache/kylin/common/util/Dictionary.java|   2 +-
 .../model/validation/rule/DictionaryRule.java   |  78 +-
 .../validation/rule/DictionaryRuleTest.java |  28 +-
 .../apache/kylin/dict/AppendTrieDictionary.java | 285 +--
 .../kylin/dict/AppendTrieDictionaryChecker.java | 102 +++
 .../org/apache/kylin/dict/CachedTreeMap.java| 260 +++---
 .../kylin/dict/GlobalDictionaryBuilder.java |  36 +-
 .../kylin/dict/AppendTrieDictionaryTest.java| 150 +++-
 .../apache/kylin/dict/CachedTreeMapTest.java| 320 +---
 .../kylin/measure/bitmap/BitmapCounterTest.java |   6 +-
 ...t_kylin_cube_without_slr_left_join_desc.json |  16 +-
 .../localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv  | 804 +--
 .../flatten_data_for_without_slr_left_join.csv  | 804 +--
 .../test_kylin_inner_join_model_desc.json   |   3 +-
 .../test_kylin_inner_join_view_model_desc.json  |   3 +-
 .../test_kylin_left_join_model_desc.json|   3 +-
 .../test_kylin_left_join_view_model_desc.json   |   3 +-
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |   8 +-
 .../source/hive/ITHiveTableReaderTest.java  |   2 +-
 .../query/sql_distinct_precisely/query00.sql|   2 +-
 .../query/sql_distinct_precisely/query01.sql|   2 +-
 .../query/sql_distinct_precisely/query02.sql|   2 +-
 .../query/sql_distinct_precisely/query03.sql|   3 +-
 .../query/sql_distinct_precisely/query04.sql|   3 +-
 .../query/sql_distinct_precisely/query05.sql|  25 -
 .../query/sql_distinct_precisely/query06.sql|  26 -
 .../query/sql_distinct_precisely/query07.sql|  24 -
 29 files changed, 1737 insertions(+), 1283 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 8068fd1..677b713 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -403,13 +403,13 @@ public class FactTableGenerator {
 }
 
 private String createRandomCell(ColumnDesc cDesc) {
-String type = cDesc.getTypeName();
-String s = type.toLowerCase();
-if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
+DataType type =cDesc.getType();
+String s = type.getName();
+if (s.equals("char") || s.equals("varchar")) {
 StringBuilder sb = new StringBuilder();
-for (int i = 0; i < 2; i++) {
-sb.append((char) ('a' + r.nextInt(10)));// there are 10*10
-// possible strings
+int len = Math.min(type.getPrecision(), 3);
+for (int i = 0; i < len; i++) {
+sb.append((char) ('a' + r.nextInt(10)));  // cardinality at 
most 10x10x10
 }
 return sb.toString();
 } else if (s.equals("bigint") || s.equals("int") || 
s.equals("tinyint") || s.equals("smallint")) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3c10826..f46c185 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -232,6 +232,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "1000"));
 }
 
+public int getAppendDictMaxVersions() {
+return 
Integer.parseInt(getOptional("kylin.dictionary.append-max-versions", "3"));
+}
+
+public int getAppendDictVersionTTL() {
+return

[13/50] [abbrv] kylin git commit: minor, massin udf reverse

2016-12-06 Thread lidong

minor, massin udf reverse

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1b90adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1b90adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1b90adb

Branch: refs/heads/master-hbase1.x
Commit: b1b90adbe896e808776caf49ad7172979cf860df
Parents: 875a593
Author: Roger Shi 
Authored: Wed Nov 30 13:35:12 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Nov 30 13:49:16 2016 +0800

--
 .../metadata/filter/UDF/MassInTupleFilter.java  | 28 +++-
 1 file changed, 27 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b1b90adb/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 3f0546c..29c5550 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -50,6 +50,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 private String filterTableName;//key in MetadataManager.extFilterMap
 private String filterTableResourceIdentifier;//HDFS path, or hbase table 
name depending on FilterTableType
 private Functions.FilterTableType filterTableType;
+private boolean reverse = false;
 
 public MassInTupleFilter() {
 super(Lists. newArrayList(), 
TupleFilter.FilterOperatorEnum.MASSIN);
@@ -66,7 +67,18 @@ public class MassInTupleFilter extends FunctionTupleFilter {
 valueProvider = 
VALUE_PROVIDER_FACTORY.getProvider(filterTableType, 
filterTableResourceIdentifier, column);
 }
 boolean ret = valueProvider.getMassInValues().contains(colValue);
-return ret;
+return reverse ? !ret : ret;
+}
+
+@Override
+public TupleFilter reverse() {
+try {
+MassInTupleFilter result = (MassInTupleFilter) this.clone();
+result.setReverse(!this.isReverse());
+return result;
+} catch (CloneNotSupportedException e) {
+throw new UnsupportedOperationException(e);
+}
 }
 
 @Override
@@ -144,4 +156,18 @@ public class MassInTupleFilter extends FunctionTupleFilter 
{
 return false;
 }
 
+public boolean isReverse() {
+return reverse;
+}
+
+public void setReverse(boolean reverse) {
+this.reverse = reverse;
+}
+
+@Override
+protected Object clone() throws CloneNotSupportedException {
+MassInTupleFilter result = new MassInTupleFilter();
+result.setReverse(this.isReverse());
+return result;
+}
 }

1 2 >

1 - 100 of 111 matches

Mail list logo