from:"mahongbin"

kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

2016-02-29 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc fbbe50d78 -> ad8d71538


KYLIN-1458: Checking the consistency of cube segment host with the environment 
after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: Add the auto fix if inconsistency exists after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: rename parameters for avoiding misleading

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad8d7153
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad8d7153
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad8d7153

Branch: refs/heads/2.0-rc
Commit: ad8d715389cbd9fd15202bc8507c8534a903cedc
Parents: fbbe50d
Author: yangzhong 
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 1 13:39:32 2016 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java|   7 +
 .../hbase/util/CubeMigrationCheckCLI.java   | 195 +++
 2 files changed, 202 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ad8d7153/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 57f362d..53cffe5 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -143,6 +143,8 @@ public class CubeMigrationCLI {
 } else {
 showOpts();
 }
+
+checkMigrationSuccess(dstConfig, cubeName, true);
 }
 
 public static void moveCube(String srcCfgUri, String dstCfgUri, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
@@ -150,6 +152,11 @@ public class CubeMigrationCLI {
 moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), 
KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, 
purgeAndDisable, overwriteIfExists, realExecute);
 }
 
+public static void checkMigrationSuccess(KylinConfig kylinConfig, String 
cubeName, Boolean ifFix) throws IOException{
+CubeMigrationCheckCLI checkCLI = new 
CubeMigrationCheckCLI(kylinConfig, ifFix);
+checkCLI.execute(cubeName);
+}
+
 private static String checkAndGetHbaseUrl() {
 String srcMetadataUrl = srcConfig.getMetadataUrl();
 String dstMetadataUrl = dstConfig.getMetadataUrl();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ad8d7153/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 000..2762561
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's 
MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * 
+ */
+public class CubeMigrationCheckCLI {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+private static final Option OPTION_FIX =

kylin git commit: KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration

2016-02-29 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5474fe4e8 -> ca5979554


KYLIN-1458: Checking the consistency of cube segment host with the environment 
after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: Add the auto fix if inconsistency exists after cube migration

Signed-off-by: Hongbin Ma 

KYLIN-1458: rename parameters for avoiding misleading

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca597955
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca597955
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca597955

Branch: refs/heads/2.x-staging
Commit: ca597955477650a388f882b968e25499b02a4bf0
Parents: 5474fe4
Author: yangzhong 
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Mar 1 13:38:02 2016 +0800

--
 .../storage/hbase/util/CubeMigrationCLI.java|   7 +
 .../hbase/util/CubeMigrationCheckCLI.java   | 195 +++
 2 files changed, 202 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index caf79b2..09aab48 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -141,6 +141,8 @@ public class CubeMigrationCLI {
 } else {
 showOpts();
 }
+
+checkMigrationSuccess(dstConfig, cubeName, true);
 }
 
 public static void moveCube(String srcCfgUri, String dstCfgUri, String 
cubeName, String projectName, String copyAcl, String purgeAndDisable, String 
overwriteIfExists, String realExecute) throws IOException, InterruptedException 
{
@@ -148,6 +150,11 @@ public class CubeMigrationCLI {
 moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), 
KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, 
purgeAndDisable, overwriteIfExists, realExecute);
 }
 
+public static void checkMigrationSuccess(KylinConfig kylinConfig, String 
cubeName, Boolean ifFix) throws IOException{
+CubeMigrationCheckCLI checkCLI = new 
CubeMigrationCheckCLI(kylinConfig, ifFix);
+checkCLI.execute(cubeName);
+}
+
 private static String checkAndGetHbaseUrl() {
 String srcMetadataUrl = srcConfig.getMetadataUrl();
 String dstMetadataUrl = dstConfig.getMetadataUrl();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca597955/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 000..2762561
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's 
MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * 
+ */
+public class CubeMigrationCheckCLI {
+
+private static final Logger logger = 
LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+private static final Option OPTION_FIX =

[2/4] kylin git commit: KYLIN-1366 simply metadata version binding

2016-02-26 Thread mahongbin

KYLIN-1366 simply metadata version binding

kylin 1366


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab9d5791
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab9d5791
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab9d5791

Branch: refs/heads/2.x-staging
Commit: ab9d57914a119b5b426d2bfae712ca55ff35c92a
Parents: 4c08ded
Author: honma 
Authored: Wed Feb 24 15:58:07 2016 +0800
Committer: honma 
Committed: Fri Feb 26 17:54:37 2016 +0800

--
 .../org/apache/kylin/common/KylinConfigBase.java   |  8 
 .../java/org/apache/kylin/common/KylinVersion.java |  2 +-
 .../common/persistence/RootPersistentEntity.java   | 17 +
 .../java/org/apache/kylin/cube/CubeInstance.java   |  5 ++---
 .../java/org/apache/kylin/cube/model/CubeDesc.java |  2 +-
 .../java/org/apache/kylin/dict/DictionaryInfo.java |  4 ++--
 .../apache/kylin/dict/lookup/SnapshotManager.java  |  2 +-
 .../apache/kylin/metadata/model/DataModelDesc.java |  2 +-
 .../kylin/metadata/project/ProjectInstance.java|  2 +-
 .../kylin/metadata/project/ProjectManager.java |  2 +-
 .../kylin/storage/hybrid/HybridInstance.java   |  2 +-
 .../org/apache/kylin/invertedindex/IIInstance.java |  2 +-
 .../kylin/rest/controller/CubeController.java  |  4 ++--
 .../storage/hbase/util/ExtendCubeToHybridCLI.java  |  4 ++--
 14 files changed, 21 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ab9d5791/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5f9983a..7707684 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -61,14 +61,6 @@ public class KylinConfigBase implements Serializable {
 return kylinHome;
 }
 
-/**
- * @see KylinVersion
- * @return current kylin version
- */
-public static String getKylinVersion(){
-return KylinVersion.getCurrentVersion();
-}
-
 // 

 
 private volatile Properties properties = new Properties();

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab9d5791/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index d711b38..42cf237 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -21,7 +21,7 @@ package org.apache.kylin.common;
  *
  * @since 2.1
  */
-class KylinVersion {
+public class KylinVersion {
 /**
  * Require MANUAL updating kylin version per ANY upgrading.
  */

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab9d5791/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index 327ddcc..c46abe7 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -26,11 +26,11 @@ import java.util.Date;
 import java.util.UUID;
 
 import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.kylin.common.KylinVersion;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.common.KylinConfig;
 
 /**
  * Marks the root entity of JSON persistence. Unit of read, write, cache, and
@@ -80,14 +80,14 @@ abstract public class RootPersistentEntity implements 
AclEntity, Serializable {
  * For example: 2.1
  */
 @JsonProperty("version")
-protected String version;
+protected String version = KylinVersion.getCurrentVersion();
 
 public String getVersion() {
-  return version;
+return version;
 }
 
 public void setVersion(String version) {
-  this.version = version;
+this.version = version;
 }
 
 public String getUuid() {
@@ -110,14 +110,7

[1/4] kylin git commit: KYLIN-1453 cuboid sharding based on specific column

2016-02-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 2e1d2f6b6 -> 294fc7078


KYLIN-1453 cuboid sharding based on specific column


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/294fc707
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/294fc707
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/294fc707

Branch: refs/heads/2.x-staging
Commit: 294fc70785ab009e6b4d8a12cbeb609d46f89a93
Parents: 3f5074e
Author: honma 
Authored: Thu Feb 25 19:02:28 2016 +0800
Committer: honma 
Committed: Fri Feb 26 17:54:37 2016 +0800

--
 .../org/apache/kylin/common/util/BytesUtil.java |  18 +-
 .../dict/TupleFilterFunctionTransformer.java| 170 +++
 .../dict/TupleFilterFunctionTranslator.java | 166 --
 .../kylin/metadata/filter/CaseTupleFilter.java  |   7 +-
 .../metadata/filter/ColumnTupleFilter.java  |  11 +-
 .../metadata/filter/CompareTupleFilter.java |  10 +-
 .../metadata/filter/ConstantTupleFilter.java|  16 +-
 .../metadata/filter/DynamicTupleFilter.java |   9 +-
 .../metadata/filter/ExtractTupleFilter.java |   7 +-
 .../metadata/filter/FunctionTupleFilter.java|  16 +-
 .../filter/ITupleFilterTransformer.java |  23 +++
 .../metadata/filter/ITupleFilterTranslator.java |  26 ---
 .../metadata/filter/LogicalTupleFilter.java |  10 +-
 .../kylin/metadata/filter/TupleFilter.java  |   5 +-
 .../metadata/filter/TupleFilterSerializer.java  |  25 ++-
 .../common/coprocessor/FilterDecorator.java |   8 +-
 .../hbase/cube/v2/CubeSegmentScanner.java   |  10 +-
 .../common/coprocessor/FilterBaseTest.java  |  46 -
 .../common/coprocessor/FilterEvaluateTest.java  |   4 +-
 .../common/coprocessor/FilterSerializeTest.java |  26 ++-
 20 files changed, 325 insertions(+), 288 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index e01ce4f..4e0701c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -57,7 +57,6 @@ public class BytesUtil {
 return integer;
 }
 
-
 public static long readLong(ByteBuffer buffer, int size) {
 long integer = 0;
 for (int i = 0; i < size; i++) {
@@ -133,11 +132,15 @@ public class BytesUtil {
 // from WritableUtils
 // 

 
+
 public static void writeVInt(int i, ByteBuffer out) {
+
 writeVLong(i, out);
+
 }
 
 public static void writeVLong(long i, ByteBuffer out) {
+
 if (i >= -112 && i <= 127) {
 out.put((byte) i);
 return;
@@ -203,6 +206,8 @@ public class BytesUtil {
 }
 
 public static void writeUnsigned(int num, int size, ByteBuffer out) {
+
+
 int mask = 0xff << ((size - 1) * 8);
 for (int i = size; i > 0; i--) {
 int v = (num & mask) >> (i - 1) * 8;
@@ -222,6 +227,7 @@ public class BytesUtil {
 }
 
 public static void writeLong(long num, ByteBuffer out) {
+
 for (int i = 0; i < 8; i++) {
 out.put((byte) num);
 num >>>= 8;
@@ -257,6 +263,8 @@ public class BytesUtil {
 }
 int len = str.length();
 BytesUtil.writeVInt(len, out);
+
+
 for (int i = 0; i < len; i++) {
 out.put((byte) str.charAt(i));
 }
@@ -335,7 +343,7 @@ public class BytesUtil {
 writeVInt(-1, out);
 return;
 }
-writeVInt(array.length, out);
+writeVInt(length, out);
 out.put(array, offset, length);
 }
 
@@ -348,7 +356,7 @@ public class BytesUtil {
 in.get(array);
 return array;
 }
-
+
 public static int peekByteArrayLength(ByteBuffer in) {
 int start = in.position();
 int arrayLen = readVInt(in);
@@ -369,6 +377,7 @@ public class BytesUtil {
 writeVInt(array.length, out);
 byte b_true = (byte) 1;
 byte b_false = (byte) 0;
+
 for (int i = 0; i < array.length; i++) {
 if (array[i])
 out.put(b_true);
@@ -428,7 +437,4 @@ public class BytesUtil {
 return sb.toString();
 }
 
-public static void main(String[] args) throws Exception {
-}
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/294fc707/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java

[3/4] kylin git commit: minor, fix CI occasionally fail issue

2016-02-26 Thread mahongbin

minor, fix CI occasionally fail issue


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f5074ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f5074ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f5074ee

Branch: refs/heads/2.x-staging
Commit: 3f5074ee1568d5b0ba50d70d5c35319cd8223cc9
Parents: ab9d579
Author: honma 
Authored: Thu Feb 25 14:13:48 2016 +0800
Committer: honma 
Committed: Fri Feb 26 17:54:37 2016 +0800

--
 .../dict/TupleFilterDictionaryTranslater.java   | 165 --
 .../dict/TupleFilterFunctionTranslator.java | 166 +++
 .../metadata/filter/FunctionTupleFilter.java|   5 +-
 .../metadata/filter/function/BuiltInMethod.java |  33 ++--
 .../cache/AbstractCacheFledgedQuery.java|  32 +---
 .../kylin/storage/cache/DynamicCacheTest.java   |  15 +-
 .../kylin/storage/cache/StaticCacheTest.java|  19 ++-
 .../kylin/storage/hbase/ITStorageTest.java  |  11 +-
 .../common/coprocessor/FilterDecorator.java |   4 +-
 .../hbase/cube/v2/CubeSegmentScanner.java   |   4 +-
 10 files changed, 226 insertions(+), 228 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
deleted file mode 100644
index 9ef360d..000
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import com.google.common.primitives.Primitives;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.FunctionTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ListIterator;
-
-/**
- * Created by dongli on 1/7/16.
- */
-public class TupleFilterDictionaryTranslater implements ITupleFilterTranslator 
{
-public static final Logger logger = 
LoggerFactory.getLogger(TupleFilterDictionaryTranslater.class);
-
-private IDictionaryAware dictionaryAware;
-
-public TupleFilterDictionaryTranslater(IDictionaryAware dictionaryAware) {
-this.dictionaryAware = dictionaryAware;
-}
-
-@Override
-public TupleFilter translate(TupleFilter tupleFilter) {
-TupleFilter translated = null;
-if (tupleFilter instanceof CompareTupleFilter) {
-translated = translateCompareTupleFilter((CompareTupleFilter) 
tupleFilter);
-if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {" 
+ translated + "}");
-}
-} else if (tupleFilter instanceof FunctionTupleFilter) {
-translated = translateFunctionTupleFilter((FunctionTupleFilter) 
tupleFilter);
-if (translated != null) {
-logger.info("Translated {" + tupleFilter + "} to IN clause: {" 
+ translated + "}");
-}
-} else if (tupleFilter instanceof LogicalTupleFilter) {
-ListIterator childIterator = 
(ListIterator) tupleFilter.getChildren().listIterator();
-while (childIterator.hasNext()) {
-TupleFilter tempTranslated =

[2/2] kylin git commit: KYLIN-1435: Relax the checking for PartitionMetadata and logger the error code

2016-02-23 Thread mahongbin

KYLIN-1435: Relax the checking for PartitionMetadata and logger the error code

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f93bec1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f93bec1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f93bec1

Branch: refs/heads/2.0-rc
Commit: 1f93bec1ca14fe0bdc2943fa9000404401f0acf5
Parents: f18f7b0
Author: yangzhong 
Authored: Tue Feb 23 17:37:11 2016 +0800
Committer: honma 
Committed: Tue Feb 23 18:22:08 2016 +0800

--
 .../java/org/apache/kylin/source/kafka/KafkaStreamingInput.java | 5 -
 .../java/org/apache/kylin/source/kafka/util/KafkaUtils.java | 5 -
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1f93bec1/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
index ee5a555..bcde47b 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
@@ -123,7 +123,10 @@ public class KafkaStreamingInput implements 
IStreamingInput {
 
 private Broker getLeadBroker() {
 final PartitionMetadata partitionMetadata = 
KafkaRequester.getPartitionMetadata(kafkaClusterConfig.getTopic(), partitionId, 
replicaBrokers, kafkaClusterConfig);
-if (partitionMetadata != null && partitionMetadata.errorCode() == 
0) {
+if (partitionMetadata != null) {
+if (partitionMetadata.errorCode() != 0){
+logger.warn("PartitionMetadata errorCode: 
"+partitionMetadata.errorCode());
+}
 replicaBrokers = partitionMetadata.replicas();
 return partitionMetadata.leader();
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1f93bec1/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
index ab54abb..f506999 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaUtils.java
@@ -51,7 +51,10 @@ public final class KafkaUtils {
 
 public static Broker getLeadBroker(KafkaClusterConfig kafkaClusterConfig, 
int partitionId) {
 final PartitionMetadata partitionMetadata = 
KafkaRequester.getPartitionMetadata(kafkaClusterConfig.getTopic(), partitionId, 
kafkaClusterConfig.getBrokers(), kafkaClusterConfig);
-if (partitionMetadata != null && partitionMetadata.errorCode() == 0) {
+if (partitionMetadata != null) {
+if (partitionMetadata.errorCode() != 0){
+logger.warn("PartitionMetadata errorCode: 
"+partitionMetadata.errorCode());
+}
 return partitionMetadata.leader();
 } else {
 return null;

[1/2] kylin git commit: KYLIN-1436: If error exists during fetching streaming messages, streaming building should throw exception

2016-02-23 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc f18f7b025 -> 924421810


KYLIN-1436: If error exists during fetching streaming messages, streaming 
building should throw exception

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92442181
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92442181
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92442181

Branch: refs/heads/2.0-rc
Commit: 9244218103906566d50bb1eb17e6dbd8da2eee6e
Parents: 1f93bec
Author: yangzhong 
Authored: Tue Feb 23 18:01:17 2016 +0800
Committer: honma 
Committed: Tue Feb 23 18:22:08 2016 +0800

--
 .../java/org/apache/kylin/source/kafka/KafkaStreamingInput.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/92442181/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
index bcde47b..2e262b3 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
@@ -90,8 +90,7 @@ public class KafkaStreamingInput implements IStreamingInput {
 logger.warn("this thread should not be interrupted, just 
ignore", e);
 continue;
 } catch (ExecutionException e) {
-logger.error("error when get StreamingMessages", 
e.getCause());
-continue;
+throw new RuntimeException("error when get 
StreamingMessages",e.getCause());
 }
 }
 final Pair timeRange = Pair.newPair(startTime, 
endTime);

kylin git commit: minor, changes on IT vm configs

2016-02-17 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging fedd47e3c -> c9e209aea


minor, changes on IT vm configs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9e209ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9e209ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9e209ae

Branch: refs/heads/2.x-staging
Commit: c9e209aeab44b49f935091d63199c7f6dbc3bb96
Parents: fedd47e
Author: honma 
Authored: Wed Feb 17 18:58:19 2016 +0800
Committer: honma 
Committed: Wed Feb 17 18:58:27 2016 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c9e209ae/pom.xml
--
diff --git a/pom.xml b/pom.xml
index d70abce..cb1e49e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -668,7 +668,7 @@
 false
 
 
--Xms1G -Xmx4G -XX:PermSize=128M 
-XX:MaxPermSize=512M
+-Xmx4G -XX:MaxPermSize=256M

kylin git commit: minor changes on documents

2016-02-16 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/document fe0d56898 -> ed810ebea


minor changes on documents


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed810ebe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed810ebe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed810ebe

Branch: refs/heads/document
Commit: ed810ebea8f06bbeeb432469866da56a43762caf
Parents: fe0d568
Author: honma 
Authored: Thu Feb 11 20:44:56 2016 +0800
Committer: honma 
Committed: Wed Feb 17 10:36:03 2016 +0800

--
 website/_dev/howto_test.md  |  35 +++
 .../_posts/blog/2016-02-03-streaming-cubing.md  |  28 +++
 website/images/develop/streaming.png| Bin 0 -> 211683 bytes
 3 files changed, 56 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ed810ebe/website/_dev/howto_test.md
--
diff --git a/website/_dev/howto_test.md b/website/_dev/howto_test.md
index 150ece8..6aaa056 100644
--- a/website/_dev/howto_test.md
+++ b/website/_dev/howto_test.md
@@ -7,21 +7,19 @@ permalink: /development/howto_test.html
 
 In general, there should be unit tests to cover individual classes; there must 
be integration test to cover end-to-end scenarios like build, merge, and query. 
Unit test must run independently (does not require an external sandbox).
 
-
-## 2.x branches
+## Test 2.x branches
 
 * `mvn test` to run unit tests, which has a limited test coverage.
 * Unit tests has no external dependency and can run on any machine.
 * The unit tests do not cover end-to-end scenarios like build, merge, and 
query.
 * The unit tests take a few minutes to complete.
 * `dev-support/test_all_against_hdp_2_2_4_2_2.sh` to run integration tests, 
which has the best test coverage.
-* Integration tests __must run on a Hadoop sandbox__. Make sure all 
changes you want to test are avaiable on sandbox.
+* Integration tests __better be run on a Hadoop sandbox__. We suggest to 
checkout a copy of code in your sandbox and direct run the 
test_all_against_hdp_2_2_4_2_2.sh in it. If you don't want to put codes on 
sandbox, refer to __More on 2.x UT/IT separation__
 * As the name indicates, the script is only for hdp 2.2.4.2, but you get 
the idea of how integration test run from it.
 * The integration tests start from generate random data, then build cube, 
merge cube, and finally query the result and compare to H2 DB.
-* The integration tests take a few hours to complete.
-
+* The integration tests take one to two hours to complete.
 
-## 1.x branches
+## Test 1.x branches
 
 * `mvn test` to run unit tests, which has a limited test coverage.
 * What's special about 1.x is that a hadoop/hbase mini cluster is used to 
cover queries in unit test.
@@ -32,12 +30,34 @@ In general, there should be unit tests to cover individual 
classes; there must b
 * `mvn test  -fae -P sandbox`
 * `mvn test  -fae  -Dtest=org.apache.kylin.query.test.IIQueryTest 
-Dhdp.version=2.2.0.0-2041 -DfailIfNoTests=false -P sandbox`
 
+## More on 2.x UT/IT separation
+
+From Kylin 2.0 you can run UT(Unit test), environment cube provision and 
IT(Integration test) separately. 
+Running `mvn verify -Dhdp.version=2.2.4.2-2`  (assume you're on your sandbox) 
is all you need to run a complete all the test suites.
+
+It will execute the following steps sequentially:
+ 
+1. Build Artifacts 
+2. Run all UTs (takes few minutes) 
+3. Provision cubes on the sandbox environment for IT uasge (takes 1~2 
hours) 
+4. Run all ITs (takes few tens of minutes) 
+5. verify jar stuff 
+
+If your code change is minor and it merely requires running UT, use: 
+`mvn test`
+If your sandbox is already provisioned and your code change will not affect 
the result of sandbox provision, (and you don't want to wait hours of 
provision) just run the following commands to separately run UT and IT: 
+`mvn test`
+`mvn failsafe:integration-test`
+
+### Cube Provision
+
+Environment cube provision is indeed running kylin cubing jobs to prepare 
example cubes in the sandbox. These prepared cubes will be used by the ITs. 
Currently provision step is bound with the maven pre-integration-test phase, 
and it contains running BuildCubeWithEngine (HBase required), 
BuildCubeWithStream(Kafka required) and BuildIIWithStream(Kafka Required). You 
can run the mvn commands on you sandbox or your develop computer. For the 
latter case you need to set kylin.job.run.as.remote.cmd=true in 
__$KYLIN_HOME/examples/test_case_data/sandbox/kylin.properties__. 
+Try appending `-DfastBuildMode=true` to mvn verify command to speed up 
provision by skipping incremental cubing. 
 
 ##

svn commit: r1728270 - in /kylin/site: ./ blog/ blog/2016/ blog/2016/02/ blog/2016/02/03/ blog/2016/02/03/streaming-cubing/ development/

2016-02-03 Thread mahongbin

Author: mahongbin
Date: Wed Feb  3 08:35:20 2016
New Revision: 1728270

URL: http://svn.apache.org/viewvc?rev=1728270=rev
Log:
UPDATE MESSAGE

Added:
kylin/site/blog/2016/
kylin/site/blog/2016/02/
kylin/site/blog/2016/02/03/
kylin/site/blog/2016/02/03/streaming-cubing/
kylin/site/blog/2016/02/03/streaming-cubing/index.html
Modified:
kylin/site/blog/index.html
kylin/site/development/about_temp_files.html
kylin/site/development/dev_env.html
kylin/site/development/howto_contribute.html
kylin/site/development/howto_docs.html
kylin/site/development/howto_package.html
kylin/site/development/howto_release.html
kylin/site/development/howto_test.html
kylin/site/development/index.html
kylin/site/development/new_metadata.html
kylin/site/development/web_tech.html
kylin/site/feed.xml

Added: kylin/site/blog/2016/02/03/streaming-cubing/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/blog/2016/02/03/streaming-cubing/index.html?rev=1728270=auto
==
--- kylin/site/blog/2016/02/03/streaming-cubing/index.html (added)
+++ kylin/site/blog/2016/02/03/streaming-cubing/index.html Wed Feb  3 08:35:20 
2016
@@ -0,0 +1,272 @@
+
+
+
+   
+
+
+  
+  
+  
+
+  Apache Kylin | Streaming cubing (Prototype)
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+  
+  
+
+  http://kylin.apache.org/blog/2016/02/03/streaming-cubing/;>
+  http://kylin.apache.org/feed.xml; />
+
+
+
+  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new 
Date();a=s.createElement(o),
+  
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+  //oringal tracker for kylin.io
+  ga('create', 'UA-55534813-1', 'auto');
+  //new tracker for kylin.apache.org
+  ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+  ga('send', 'pageview');
+  ga('toplevel.send', 'pageview');
+
+
+
+
+ 
+ 
+
+
+   
+   
+
+
+  
+  
+  Apache Kylinâ¢ Extreme OLAP Engine for Big 
Data 
+
+  
+  
+
+  
+  
+  
+
+
+
+  
+Toggle navigation
+
+
+
+  
+ 
+
+
+
+
+  
+ Home
+  Docs
+  Download
+  Community
+  Development
+  Blog
+  ä¸æç  
+  https://twitter.com/apachekylin; target="_blank" 
class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" >
+  https://github.com/apache/kylin; target="_blank" 
class="fa fa-github-alt fa-lg" title="Github: apache/kylin" >  
+  https://www.facebook.com/kylinio; target="_blank" 
class="fa fa-facebook fa-lg" title="Facebook: kylin.io" >   
+
+
+  
+
+ 
+
+   
+   
+   
+   Apache Kylinâ¢ 
Technical Blog
+   
+   
+
+   
+   
+
+   
+
+
+
+  
+Streaming cubing (Prototype)
+Feb 3, 2016 â¢ Hongbin Ma
+  
+
+  
+One of the most important features in 2.x branches is streaming cubing 
which enables OLAP analysis on streaming data. Streaming cubing delivers faster 
insights on the data to help more promptly business decisions. Even though 
there are already many real time analysis tools in open source community, Kylin 
Streaming cubing still differs from them in multiple angles:
+
+Firstly, Kylin Streaming Cubing aligns with Kylin traditional cubing to 
provided unified, ANSI SQL interface. Actually Kylin Streaming shares the 
storage engine and query engine with traditional Kylin cubes, so in theory all 
of the optimization techniques to save storage and speed up query performance 
can also be applied on streaming cubes. Besides, all the supported 
aggregations/filters/UDFs still work for streaming cubes. By unifying the 
storage engine and query engine we also get freed from double amount of 
maintaince work.
+
+Secondly, Kylin Streaming Cubing does not require large amount of memory to 
store real time data, nor does it attempts to provide truly âreal timeâ 
analysis. By our customer survey we found that minutes of visualization latency 
is acceptable for OLAP analysts. So our streaming cubing adopts the micro batch 
approach. Incoming streaming data are partitioned into different time windows 
and we build a micro batch for each time window. The cube output for each micro 
batch is directly saved to HBase. The query engine goes to HBase for data 
retrieving instead of the data ingestion server. The benefit of such design is 
that we donât have to maintain large amount of in-memory index which cou

svn commit: r1728267 - in /kylin/site: ./ development/ images/develop/

2016-02-03 Thread mahongbin

Author: mahongbin
Date: Wed Feb  3 08:26:52 2016
New Revision: 1728267

URL: http://svn.apache.org/viewvc?rev=1728267=rev
Log:
UPDATE MESSAGE

Added:
kylin/site/development/streaming_cubing.html
kylin/site/images/develop/streaming.png   (with props)
Modified:
kylin/site/development/about_temp_files.html
kylin/site/development/dev_env.html
kylin/site/development/howto_contribute.html
kylin/site/development/howto_docs.html
kylin/site/development/howto_package.html
kylin/site/development/howto_release.html
kylin/site/development/howto_test.html
kylin/site/development/index.html
kylin/site/development/new_metadata.html
kylin/site/development/web_tech.html
kylin/site/feed.xml

Modified: kylin/site/development/about_temp_files.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/about_temp_files.html?rev=1728267=1728266=1728267=diff
==
--- kylin/site/development/about_temp_files.html (original)
+++ kylin/site/development/about_temp_files.html Wed Feb  3 08:26:52 2016
@@ -405,6 +405,8 @@
 
   
 
+  
+
   Kylin Web Summary  
   
 
@@ -424,6 +426,39 @@
   
 
 
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Streaming cubing 
(Prototype)  
+  
+
+
 
 
 

Modified: kylin/site/development/dev_env.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/dev_env.html?rev=1728267=1728266=1728267=diff
==
--- kylin/site/development/dev_env.html (original)
+++ kylin/site/development/dev_env.html Wed Feb  3 08:26:52 2016
@@ -405,6 +405,8 @@
 
   
 
+  
+
   Kylin Web Summary  
   
 
@@ -424,6 +426,39 @@
   
 
 
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Streaming cubing 
(Prototype)  
+  
+
+
 
 
 

Modified: kylin/site/development/howto_contribute.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_contribute.html?rev=1728267=1728266=1728267=diff
==
--- kylin/site/development/howto_contribute.html (original)
+++ kylin/site/development/howto_contribute.html Wed Feb  3 08:26:52 2016
@@ -405,6 +405,8 @@
 
   
 
+  
+
   Kylin Web Summary  
   
 
@@ -424,6 +426,39 @@
   
 
 
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Streaming cubing 
(Prototype)  
+  
+
+
 
 
 

Modified: kylin/site/development/howto_docs.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_docs.html?rev=1728267=1728266=1728267=diff
==
--- kylin/site/development/howto_docs.html (original)
+++ kylin/site/development/howto_docs.html Wed Feb  3 08:26:52 2016
@@ -405,6 +405,8 @@
 
   
 
+  
+
   Kylin Web Summary  
   
 
@@ -424,6 +426,39 @@
   
 
 
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Streaming cubing 
(Prototype)  
+  
+
+
 
 
 

Modified: kylin/site/development/howto_package.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_package.html?rev=1728267=1728266=1728267=diff
==
--- kylin/site/development/howto_package.html (original)
+++ kylin/site/development/howto_package.html Wed Feb  3 08:26:52 2016
@@ -405,6 +405,8 @@
 
   
 
+  
+
   Kylin Web Summary  
   
 
@@ -424,6 +426,39 @@
   
 
 
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Streaming cubing 
(Prototype)  
+  
+
+
 
 
 

Modified: kylin/site/development/howto_release.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_release.html?rev=1728267=1728266=1728267=diff
==
--- kylin/site/development/howto_release.html (original)
+++ kylin/site/development/howto_release.html Wed Feb  3 08:26:52 2016
@@ -405,6 +405,8 @@
 
   
 
+  
+
   Kylin Web Summary  
   
 
@@ -424,6 +426,39 @@
   
 
 
+   
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Streaming cubing 
(Prototype)  
+  
+
+
 
 
 

Modified: kylin/site/development/howto_test.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_test.html?rev=1728267=1728266=1728267=diff

kylin git commit: KYLIN-1366 Bind metadata version with release version(chen hao)

2016-02-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 36e26528e -> e2a0d1d82


KYLIN-1366 Bind metadata version with release version(chen hao)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2a0d1d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2a0d1d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2a0d1d8

Branch: refs/heads/2.x-staging
Commit: e2a0d1d821b0650a201370e0964a43207a744c21
Parents: 36e2652
Author: honma 
Authored: Thu Feb 4 10:06:47 2016 +0800
Committer: honma 
Committed: Thu Feb 4 10:06:47 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  8 
 .../org/apache/kylin/common/KylinVersion.java   | 40 
 .../persistence/RootPersistentEntity.java   | 36 +++---
 .../org/apache/kylin/cube/CubeInstance.java | 14 +--
 .../org/apache/kylin/cube/model/CubeDesc.java   |  2 +-
 .../org/apache/kylin/dict/DictionaryInfo.java   |  4 +-
 .../kylin/dict/lookup/SnapshotManager.java  |  2 +-
 .../kylin/metadata/model/DataModelDesc.java |  2 +-
 .../kylin/metadata/project/ProjectInstance.java |  2 +-
 .../kylin/metadata/project/ProjectManager.java  |  2 +-
 .../kylin/storage/hybrid/HybridInstance.java|  2 +-
 .../metadata/cube/kylin_sales_cube.json |  2 +-
 .../cube_desc/kylin_sales_cube_desc.json|  1 +
 .../metadata/model_desc/kylin_sales_model.json  |  1 +
 .../metadata/project/learn_kylin.json   |  1 +
 .../metadata/table/DEFAULT.KYLIN_CAL_DT.json|  1 +
 .../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json |  1 +
 .../metadata/table/DEFAULT.KYLIN_SALES.json |  1 +
 .../test_kylin_cube_with_slr_1_new_segment.json |  2 +-
 .../cube/test_kylin_cube_with_slr_empty.json|  2 +-
 ...est_kylin_cube_with_slr_left_join_empty.json |  2 +-
 ...est_kylin_cube_with_slr_left_join_ready.json |  2 +-
 .../cube/test_kylin_cube_with_slr_ready.json|  2 +-
 ...st_kylin_cube_with_slr_ready_2_segments.json |  2 +-
 .../cube/test_kylin_cube_without_slr_empty.json |  2 +-
 ..._kylin_cube_without_slr_left_join_empty.json |  2 +-
 ..._kylin_cube_without_slr_left_join_ready.json |  2 +-
 ..._without_slr_left_join_ready_2_segments.json |  2 +-
 .../cube/test_kylin_cube_without_slr_ready.json |  2 +-
 .../cube/test_streaming_table_cube.json |  2 +-
 .../test_kylin_cube_with_slr_desc.json  |  1 +
 ...test_kylin_cube_with_slr_left_join_desc.json |  1 +
 .../test_kylin_cube_without_slr_desc.json   |  2 +
 ...t_kylin_cube_without_slr_left_join_desc.json |  1 +
 .../test_streaming_table_cube_desc.json |  1 +
 .../hybrid/test_kylin_hybrid_inner_join.json|  1 +
 .../hybrid/test_kylin_hybrid_left_join.json |  1 +
 .../hybrid/test_kylin_hybrid_ready.json |  1 +
 .../invertedindex/test_kylin_ii_inner_join.json |  1 +
 .../invertedindex/test_kylin_ii_left_join.json  |  2 +-
 .../invertedindex/test_streaming_table_ii.json  |  2 +-
 .../test_kylin_ii_inner_join_desc.json  |  1 +
 .../test_kylin_ii_left_join_desc.json   |  1 +
 .../test_streaming_table_ii_desc.json   |  1 +
 .../localmeta/kafka/kafka_test.json |  1 +
 .../kafka/test_streaming_table_cube.json|  1 +
 .../kafka/test_streaming_table_ii.json  |  1 +
 .../test_kylin_inner_join_model_desc.json   |  1 +
 .../test_kylin_left_join_model_desc.json|  1 +
 .../test_streaming_table_model_desc.json|  1 +
 .../localmeta/project/default.json  |  1 +
 .../localmeta/streaming/kafka_test.json |  1 +
 .../streaming/test_streaming_table_cube.json|  1 +
 .../streaming/test_streaming_table_ii.json  |  1 +
 .../table/DEFAULT.STREAMING_TABLE.json  |  1 +
 .../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json  |  1 +
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |  1 +
 .../localmeta/table/EDW.TEST_CAL_DT.json|  1 +
 .../table/EDW.TEST_SELLER_TYPE_DIM.json |  1 +
 .../localmeta/table/EDW.TEST_SITES.json |  1 +
 .../apache/kylin/invertedindex/IIInstance.java  | 12 +-
 .../kylin/rest/controller/CubeController.java   |  4 +-
 .../hbase/util/ExtendCubeToHybridCLI.java   |  4 +-
 63 files changed, 143 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a0d1d8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index bfad306..a44831d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java

kylin git commit: minor, avoid verbose logging in serialier

2016-02-02 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 05df71369 -> 62a8cb268


minor, avoid verbose logging in serialier


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62a8cb26
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62a8cb26
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62a8cb26

Branch: refs/heads/1.x-staging
Commit: 62a8cb268e906738502338c7978b39bb3fd2c28d
Parents: 05df713
Author: honma 
Authored: Wed Feb 3 14:40:55 2016 +0800
Committer: honma 
Committed: Wed Feb 3 14:40:55 2016 +0800

--
 .../org/apache/kylin/metadata/datatype/BigDecimalSerializer.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/62a8cb26/metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
 
b/metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
index 468e4b0..25805da 100644
--- 
a/metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
+++ 
b/metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
@@ -48,8 +48,8 @@ public class BigDecimalSerializer extends 
DataTypeSerializer {
 @Override
 public void serialize(BigDecimal value, ByteBuffer out) {
 if (value.scale() > type.getScale()) {
-if (avoidVerbose % 1 == 0) {
-logger.warn("value's scale has exceeded the " + 
type.getScale() + ", cut it off, to ensure encoded value do not exceed 
maxLength " + maxLength + " times:" + (avoidVerbose++));
+if (avoidVerbose++ % 1 == 0) {
+logger.warn("value's scale has exceeded the " + 
type.getScale() + ", cut it off, to ensure encoded value do not exceed 
maxLength " + maxLength + " times:" + (avoidVerbose));
 }
 value = value.setScale(type.getScale(), 
BigDecimal.ROUND_HALF_EVEN);
 }

kylin git commit: KYLIN-1313 fix CI

2016-02-01 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 82c6d588f -> 54e491502


KYLIN-1313 fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/54e49150
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/54e49150
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/54e49150

Branch: refs/heads/2.x-staging
Commit: 54e491502fb00b88fbabe0dcbc32aea91dc874e3
Parents: 82c6d58
Author: honma 
Authored: Mon Feb 1 22:14:41 2016 +0800
Committer: honma 
Committed: Mon Feb 1 22:14:41 2016 +0800

--
 .../measure/extendedcolumn/ExtendedColumnSerializerTest.java   | 6 +++---
 examples/test_case_data/sandbox/kylin.properties   | 2 +-
 .../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/54e49150/core-metadata/src/test/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnSerializerTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnSerializerTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnSerializerTest.java
index dd73369..fed1c28 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnSerializerTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnSerializerTest.java
@@ -39,7 +39,7 @@ public class ExtendedColumnSerializerTest {
 public void testSerDesNull() {
 ExtendedColumnSerializer serializer = new 
ExtendedColumnSerializer(DataType.getType("extendedcolumn(20)"));
 MeasureIngester ingester = measureType.newIngester();
-ByteArray array = ingester.valueOf(new String[] { null }, null, null);
+ByteArray array = ingester.valueOf(new String[] { null, null }, null, 
null);
 Assert.assertTrue(new ByteArray().equals(array));
 
 ByteBuffer buffer = ByteBuffer.allocate(serializer.maxLength());
@@ -57,7 +57,7 @@ public class ExtendedColumnSerializerTest {
 
 ExtendedColumnSerializer serializer = new 
ExtendedColumnSerializer(DataType.getType("extendedcolumn(20)"));
 MeasureIngester ingester = measureType.newIngester();
-ByteArray array = ingester.valueOf(new String[] { text }, null, null);
+ByteArray array = ingester.valueOf(new String[] { null, text }, null, 
null);
 
 ByteBuffer buffer = ByteBuffer.allocate(serializer.maxLength());
 serializer.serialize(array, buffer);
@@ -71,7 +71,7 @@ public class ExtendedColumnSerializerTest {
 String text = StringUtils.repeat("h", 21);
 ExtendedColumnSerializer serializer = new 
ExtendedColumnSerializer(DataType.getType("extendedcolumn(20)"));
 MeasureIngester ingester = measureType.newIngester();
-ByteArray array = ingester.valueOf(new String[] { text }, null, null);
+ByteArray array = ingester.valueOf(new String[] { null, text }, null, 
null);
 
 ByteBuffer buffer = ByteBuffer.allocate(serializer.maxLength());
 serializer.serialize(array, buffer);

http://git-wip-us.apache.org/repos/asf/kylin/blob/54e49150/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index ccdc3a1..bf161fc 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -24,7 +24,7 @@ kylin.job.mapreduce.default.reduce.input.mb=500
 
 # If true, job engine will not assume that hadoop CLI reside on the same 
server as it self
 # you will have to specify kylin.job.remote.cli.hostname, 
kylin.job.remote.cli.username and kylin.job.remote.cli.password
-kylin.job.run.as.remote.cmd=true
+kylin.job.run.as.remote.cmd=false
 
 # Only necessary when kylin.job.run.as.remote.cmd=true
 kylin.job.remote.cli.hostname=sandbox

http://git-wip-us.apache.org/repos/asf/kylin/blob/54e49150/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 89a9740..fa71db2 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -45,7 +45,7 @@ public class ITKylinQueryTest extends KylinTestBase {
 
 @BeforeClass
 public static void setUp() throws Exception {
-

kylin git commit: KYLIN-1372 fix CI

2016-01-31 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 75970c3b1 -> 555a2e566


KYLIN-1372 fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/555a2e56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/555a2e56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/555a2e56

Branch: refs/heads/1.x-staging
Commit: 555a2e5667837d6bdf16460a34c88f3f5130bf31
Parents: 75970c3
Author: honma 
Authored: Mon Feb 1 13:48:06 2016 +0800
Committer: honma 
Committed: Mon Feb 1 13:48:06 2016 +0800

--
 query/src/test/resources/query/sql_dynamic/query03.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/555a2e56/query/src/test/resources/query/sql_dynamic/query03.sql
--
diff --git a/query/src/test/resources/query/sql_dynamic/query03.sql 
b/query/src/test/resources/query/sql_dynamic/query03.sql
index 31d5d0b..3187c33 100644
--- a/query/src/test/resources/query/sql_dynamic/query03.sql
+++ b/query/src/test/resources/query/sql_dynamic/query03.sql
@@ -17,7 +17,7 @@
 --
 
 select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct leaf_categ_id) as LEAF_CATEG_CNT
+ , count(1) as TRANS_CNT
  from test_kylin_fact
  inner JOIN edw.test_cal_dt as test_cal_dt
  ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt

kylin git commit: KYLIN-1372 fix CI

2016-01-31 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc a9637c9c5 -> d3250acf3


KYLIN-1372 fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3250acf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3250acf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3250acf

Branch: refs/heads/2.0-rc
Commit: d3250acf3516ceffc08f830ecac2993cdc1a91fd
Parents: a9637c9
Author: honma 
Authored: Mon Feb 1 13:44:02 2016 +0800
Committer: honma 
Committed: Mon Feb 1 13:45:58 2016 +0800

--
 kylin-it/src/test/resources/query/sql_dynamic/query03.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d3250acf/kylin-it/src/test/resources/query/sql_dynamic/query03.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_dynamic/query03.sql 
b/kylin-it/src/test/resources/query/sql_dynamic/query03.sql
index 90574f7..ceefe70 100644
--- a/kylin-it/src/test/resources/query/sql_dynamic/query03.sql
+++ b/kylin-it/src/test/resources/query/sql_dynamic/query03.sql
@@ -17,7 +17,7 @@
 --
 
 select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct test_kylin_fact.leaf_categ_id) as 
LEAF_CATEG_CNT
+ , count(1) as TRANS_CNT
  from test_kylin_fact
  inner JOIN edw.test_cal_dt as test_cal_dt
  ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt

[5/6] kylin git commit: minor, move all hll and topn related code to core-metadata

2016-01-27 Thread mahongbin

minor, move all hll and topn related code to core-metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/daa294b6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/daa294b6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/daa294b6

Branch: refs/heads/2.x-staging
Commit: daa294b679cf857422acce9cf2c86b6c950a5b67
Parents: 5f4c581
Author: honma 
Authored: Wed Jan 27 22:19:00 2016 +0800
Committer: honma 
Committed: Wed Jan 27 22:19:00 2016 +0800

--
 core-common/pom.xml |   6 +-
 .../common/hll/HyperLogLogPlusCounter.java  | 340 ---
 .../kylin/common/hll/HyperLogLogPlusTable.java  | 199 ---
 .../org/apache/kylin/common/topn/Counter.java   |  83 -
 .../common/topn/DoubleDeltaSerializer.java  | 193 ---
 .../kylin/common/topn/DoublyLinkedList.java | 141 
 .../org/apache/kylin/common/topn/ListNode2.java |  51 ---
 .../apache/kylin/common/topn/TopNCounter.java   | 335 --
 .../common/hll/HyperLogLogCounterTest.java  | 245 -
 .../common/topn/DoubleDeltaSerializerTest.java  |  85 -
 .../kylin/common/topn/TopNCounterBasicTest.java | 132 ---
 .../common/topn/TopNCounterCombinationTest.java |  62 
 .../kylin/common/topn/TopNCounterTest.java  | 306 -
 .../cube/inmemcubing/InMemCubeBuilder.java  |   4 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |   2 +-
 .../apache/kylin/gridtable/UnitTestSupport.java |   2 +-
 .../gridtable/AggregationCacheMemSizeTest.java  |   2 +-
 .../metadata/measure/MeasureCodecTest.java  |  18 +-
 core-metadata/pom.xml   |   5 +
 .../kylin/measure/hllc/HLLCAggregator.java  |   1 -
 .../kylin/measure/hllc/HLLCMeasureType.java |   1 -
 .../kylin/measure/hllc/HLLCSerializer.java  |   1 -
 .../measure/hllc/HLLDistinctCountAggFunc.java   |   1 -
 .../measure/hllc/HyperLogLogPlusCounter.java| 340 +++
 .../measure/hllc/HyperLogLogPlusTable.java  | 199 +++
 .../org/apache/kylin/measure/topn/Counter.java  |  83 +
 .../measure/topn/DoubleDeltaSerializer.java | 193 +++
 .../kylin/measure/topn/DoublyLinkedList.java| 141 
 .../apache/kylin/measure/topn/ListNode2.java|  51 +++
 .../kylin/measure/topn/TopNAggregator.java  |   1 -
 .../apache/kylin/measure/topn/TopNCounter.java  | 335 ++
 .../measure/topn/TopNCounterSerializer.java |   3 -
 .../kylin/measure/topn/TopNMeasureType.java |   2 -
 .../topn/TopNCounterSerializerTest.java |   2 +-
 .../measure/hll/HyperLogLogCounterTest.java | 245 +
 .../measure/topn/DoubleDeltaSerializerTest.java |  85 +
 .../measure/topn/TopNCounterBasicTest.java  | 131 +++
 .../topn/TopNCounterCombinationTest.java|  62 
 .../kylin/measure/topn/TopNCounterTest.java | 306 +
 .../kylin/engine/mr/common/CubeStatsReader.java |   2 +-
 .../kylin/engine/mr/common/CuboidStatsUtil.java |   2 +-
 .../mr/steps/FactDistinctColumnsReducer.java|   2 +-
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   2 +-
 .../engine/mr/steps/MergeStatisticsStep.java|   2 +-
 .../kylin/engine/mr/steps/CubeSamplingTest.java |   3 +-
 .../steps/FactDistinctColumnsReducerTest.java   |   2 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |   3 +-
 .../engine/streaming/IStreamingOutput.java  |   2 +-
 .../streaming/OneOffStreamingBuilder.java   |   2 +-
 .../engine/streaming/StreamingBatchBuilder.java |   2 +-
 .../streaming/cube/StreamingCubeBuilder.java|   2 +-
 .../invertedindex/measure/FixedHLLCodec.java|   2 +-
 .../cardinality/ColumnCardinalityMapper.java|   2 +-
 .../cardinality/ColumnCardinalityReducer.java   |   2 +-
 .../ColumnCardinalityReducerTest.java   |   4 +-
 .../endpoint/EndpointAggregators.java   |   2 +-
 .../hbase/steps/HBaseStreamingOutput.java   |   2 +-
 57 files changed, 2210 insertions(+), 2224 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/daa294b6/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 28566d6..bfdc662 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -109,11 +109,7 @@
 
 
 
-
-org.apache.commons
-commons-math3
-test
-
+   
 
 junit
 junit

http://git-wip-us.apache.org/repos/asf/kylin/blob/daa294b6/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
--
diff --git

[4/6] kylin git commit: minor, move all hll and topn related code to core-metadata

2016-01-27 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/daa294b6/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
 
b/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
deleted file mode 100644
index 25ee404..000
--- 
a/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusTable.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.hll;
-
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-/**
- * @author xjiang
- * 
- */
-public class HyperLogLogPlusTable {
-
-// threshold and bias data taken from google's bias correction data set:
-// 
https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
-public static final double[] thresholdData = { 10, 20, 40, 80, 220, 400, 
900, 1800, 3100, 6500, 15500, 2, 5, 12, 35 };
-
-public static final double[][] rawEstimateData = {
-// precision 4
-{ 11, 11.717, 12.207, 12.7896, 13.2882, 13.8204, 14.3772, 14.9342, 
15.5202, 16.161, 16.7722, 17.4636, 18.0396, 18.6766, 19.3566, 20.0454, 20.7936, 
21.4856, 22.2666, 22.9946, 23.766, 24.4692, 25.3638, 26.0764, 26.7864, 27.7602, 
28.4814, 29.433, 30.2926, 31.0664, 31.9996, 32.7956, 33.5366, 34.5894, 35.5738, 
36.2698, 37.3682, 38.0544, 39.2342, 40.0108, 40.7966, 41.9298, 42.8704, 
43.6358, 44.5194, 45.773, 46.6772, 47.6174, 48.4888, 49.3304, 50.2506, 51.4996, 
52.3824, 53.3078, 54.3984, 55.5838, 56.6618, 57.2174, 58.3514, 59.0802, 
60.1482, 61.0376, 62.3598, 62.8078, 63.9744, 64.914, 65.781, 67.1806, 68.0594, 
68.8446, 69.7928, 70.8248, 71.8324, 72.8598, 73.6246, 74.7014, 75.393, 76.6708, 
77.2394, },
-// precision 5
-{ 23, 23.1194, 23.8208, 24.2318, 24.77, 25.2436, 25.7774, 26.2848, 
26.8224, 27.3742, 27.9336, 28.503, 29.0494, 29.6292, 30.2124, 30.798, 31.367, 
31.9728, 32.5944, 33.217, 33.8438, 34.3696, 35.0956, 35.7044, 36.324, 37.0668, 
37.6698, 38.3644, 39.049, 39.6918, 40.4146, 41.082, 41.687, 42.5398, 43.2462, 
43.857, 44.6606, 45.4168, 46.1248, 46.9222, 47.6804, 48.447, 49.3454, 49.9594, 
50.7636, 51.5776, 52.331, 53.19, 53.9676, 54.7564, 55.5314, 56.4442, 57.3708, 
57.9774, 58.9624, 59.8796, 60.755, 61.472, 62.2076, 63.1024, 63.8908, 64.7338, 
65.7728, 66.629, 67.413, 68.3266, 69.1524, 70.2642, 71.1806, 72.0566, 72.9192, 
73.7598, 74.3516, 75.5802, 76.4386, 77.4916, 78.1524, 79.1892, 79.8414, 
80.8798, 81.8376, 82.4698, 83.7656, 84.331, 85.5914, 86.6012, 87.7016, 88.5582, 
89.3394, 90.3544, 91.4912, 92.308, 93.3552, 93.9746, 95.2052, 95.727, 97.1322, 
98.3944, 98.7588, 100.242, 101.1914, 102.2538, 102.8776, 103.6292, 105.1932, 
105.9152, 107.0868, 107.6728, 108.7144, 110.3114, 110.8716,
-111.245, 112.7908, 113.7064, 114.636, 115.7464, 116.1788, 
117.7464, 118.4896, 119.6166, 120.5082, 121.7798, 122.9028, 123.4426, 124.8854, 
125.705, 126.4652, 128.3464, 128.3462, 130.0398, 131.0342, 131.0042, 132.4766, 
133.511, 134.7252, 135.425, 136.5172, 138.0572, 138.6694, 139.3712, 140.8598, 
141.4594, 142.554, 143.4006, 144.7374, 146.1634, 146.8994, 147.605, 147.9304, 
149.1636, 150.2468, 151.5876, 152.2096, 153.7032, 154.7146, 155.807, 156.9228, 
157.0372, 158.5852, },
-// precision 6
-{ 46, 46.1902, 47.271, 47.8358, 48.8142, 49.2854, 50.317, 51.354, 
51.8924, 52.9436, 53.4596, 54.5262, 55.6248, 56.1574, 57.2822, 57.837, 58.9636, 
60.074, 60.7042, 61.7976, 62.4772, 63.6564, 64.7942, 65.5004, 66.686, 67.291, 
68.5672, 69.8556, 70.4982, 71.8204, 72.4252, 73.7744, 75.0786, 75.8344, 
77.0294, 77.8098, 79.0794, 80.5732, 81.1878, 82.5648, 83.2902, 84.6784, 
85.3352, 86.8946, 88.3712, 89.0852, 90.499, 91.2686, 92.6844, 94.2234, 94.9732, 
96.3356, 97.2286, 98.7262, 100.3284, 101.1048, 102.5962, 103.3562, 105.1272, 
106.4184, 107.4974, 109.0822, 109.856, 111.48, 113.2834, 114.0208, 115.637, 
116.5174, 118.0576, 119.7476, 120.427, 122.1326, 123.2372, 125.2788, 126.6776, 
127.7926, 129.1952, 129.9564, 131.6454,

[6/6] kylin git commit: minor, check hdfs permission at build cube

2016-01-27 Thread mahongbin

minor, check hdfs permission at build cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f4667226
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f4667226
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f4667226

Branch: refs/heads/2.x-staging
Commit: f4667226d1ba34471c27192b95a0494f5f991d88
Parents: daa294b
Author: honma 
Authored: Thu Jan 28 10:34:25 2016 +0800
Committer: honma 
Committed: Thu Jan 28 10:34:25 2016 +0800

--
 .../kylin/provision/BuildCubeWithEngine.java | 19 +++
 1 file changed, 19 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f4667226/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 32facf1..f44fc38 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.provision;
 
 import java.io.File;
+import java.io.IOException;
 import java.lang.reflect.Method;
 import java.text.SimpleDateFormat;
 import java.util.List;
@@ -32,6 +33,9 @@ import java.util.concurrent.Future;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractKylinTestCase;
@@ -43,6 +47,7 @@ import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.CubeUpdate;
 import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.job.DeployUtil;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.job.execution.AbstractExecutable;
@@ -93,6 +98,20 @@ public class BuildCubeWithEngine {
 }
 
 
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+try {
+//check hdfs permission
+Configuration hconf = HadoopUtil.getCurrentConfiguration();
+FileSystem fileSystem = FileSystem.get(hconf);
+String hdfsWorkingDirectory = 
KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
+Path coprocessorDir = new Path(hdfsWorkingDirectory);
+boolean success = fileSystem.mkdirs(coprocessorDir);
+if (!success) {
+throw new IOException("mkdir fails");
+}
+} catch (IOException e) {
+throw new RuntimeException("failed to create 
kylin.hdfs.working.dir, Please make sure the user has right to access " + 
KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(), e);
+}
 }
 
 public void before() throws Exception {

[1/6] kylin git commit: minor, move all hll and topn related code to core-metadata

2016-01-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5f4c581e6 -> f4667226d


http://git-wip-us.apache.org/repos/asf/kylin/blob/daa294b6/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoublyLinkedList.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoublyLinkedList.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoublyLinkedList.java
new file mode 100644
index 000..03e4066
--- /dev/null
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoublyLinkedList.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.topn;
+
+
+/**
+ * Modified from DoublyLinkedList.java in https://github.com/addthis/stream-lib
+ * 
+ * @param 
+ */
+public class DoublyLinkedList {
+
+private int size = 0;
+private ListNode2 tail;
+private ListNode2 head;
+
+/**
+ * Append to head of list
+ */
+public ListNode2 add(T value) {
+ListNode2 node = new ListNode2(value);
+add(node);
+
+return node;
+}
+
+/**
+ * Prepend to tail of list
+ */
+public ListNode2 enqueue(T value) {
+ListNode2 node = new ListNode2(value);
+   
+return enqueue(node);
+}
+
+public ListNode2 enqueue(ListNode2 node) {
+if (size++ == 0) {
+head = node;
+} else {
+node.next = tail;
+tail.prev = node;
+}
+
+tail = node;
+
+return node;
+}
+
+public void add(ListNode2 node) {
+node.prev = head;
+node.next = null;
+
+if (size++ == 0) {
+tail = node;
+} else {
+head.next = node;
+}
+
+head = node;
+}
+
+public ListNode2 addAfter(ListNode2 node, T value) {
+ListNode2 newNode = new ListNode2(value);
+addAfter(node, newNode);
+return newNode;
+}
+
+public void addAfter(ListNode2 node, ListNode2 newNode) {
+newNode.next = node.next;
+newNode.prev = node;
+node.next = newNode;
+if (newNode.next == null) {
+head = newNode;
+} else {
+newNode.next.prev = newNode;
+}
+size++;
+}
+
+
+public void addBefore(ListNode2 node, ListNode2 newNode) {
+newNode.prev = node.prev;
+newNode.next = node;
+node.prev = newNode;
+if (newNode.prev == null) {
+tail = newNode;
+} else {
+newNode.prev.next = newNode;
+}
+size++;
+}
+
+public void remove(ListNode2 node) {
+if (node == tail) {
+tail = node.next;
+} else {
+node.prev.next = node.next;
+}
+
+if (node == head) {
+head = node.prev;
+} else {
+node.next.prev = node.prev;
+}
+size--;
+}
+
+public int size() {
+return size;
+}
+
+
+public ListNode2 head() {
+return head;
+}
+
+public ListNode2 tail() {
+return tail;
+}
+
+public boolean isEmpty() {
+return size == 0;
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/daa294b6/core-metadata/src/main/java/org/apache/kylin/measure/topn/ListNode2.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/ListNode2.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/ListNode2.java
new file mode 100644
index 000..b2f47c9
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/ListNode2.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *

kylin git commit: KYLIN-1355 Fix package version bug and add script to build a java-hotfix package, instead of a full package

2016-01-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 91ae3fc84 -> 5f4c581e6


KYLIN-1355 Fix package version bug and add script to build a java-hotfix 
package, instead of a full package

Changes:
* Add build/script/package_libs.sh  to build library jars only tarball
* Fix version parsing bug in build/script/package.sh

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f4c581e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f4c581e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f4c581e

Branch: refs/heads/2.x-staging
Commit: 5f4c581e682835e931a7716adec7f02cfa1a51c3
Parents: 91ae3fc
Author: Hao Chen 
Authored: Wed Jan 27 17:44:11 2016 +0800
Committer: honma 
Committed: Wed Jan 27 21:53:32 2016 +0800

--
 build/script/package.sh  |   2 +-
 build/script/package_libs.sh | 100 ++
 2 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f4c581e/build/script/package.sh
--
diff --git a/build/script/package.sh b/build/script/package.sh
index a505501..53c2908 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -50,7 +50,7 @@ fi
 
 dir=$(dirname ${0})
 cd ${dir}/../..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate 
-Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+'`
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate 
-Dexpression=project.version | grep -v '\['`
 echo "kylin version: ${version}"
 export version
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5f4c581e/build/script/package_libs.sh
--
diff --git a/build/script/package_libs.sh b/build/script/package_libs.sh
new file mode 100755
index 000..71c1e8c
--- /dev/null
+++ b/build/script/package_libs.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo "Checking maven..."
+
+if [ -z "$(command -v mvn)" ]
+then
+echo "Please install maven first so that Kylin packaging can proceed"
+exit 1
+else
+echo "maven check passed"
+fi
+
+echo "Checking git..."
+
+if [ -z "$(command -v git)" ]
+then
+echo "Please install git first so that Kylin packaging can proceed"
+exit 1
+else
+echo "git check passed"
+fi
+
+dir=$(dirname ${0})
+cd ${dir}/../..
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate 
-Dexpression=project.version | grep -v '\['`
+
+if [ "$version" == "" ];then
+   echo "Failed to identify kylin version (current: `pwd`)"
+   exit 1
+fi
+
+echo "kylin version: ${version}"
+export version
+
+#commit id
+cat << EOF > build/commit_SHA1
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+EOF
+git rev-parse HEAD >> build/commit_SHA1
+
+echo "package libraries"
+mvn clean install -DskipTests   || { exit 1; }
+
+echo "copy libraries"
+rm -rf build/lib
+mkdir build/lib
+cp assembly/target/kylin-assembly-${version}-job.jar 
build/lib/kylin-job-${version}.jar
+cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar 
build/lib/kylin-coprocessor-${version}.jar
+cp

[3/3] kylin git commit: KYLIN-1355 Fix package version bug and add script to build a java-hotfix package, instead of a full package

2016-01-27 Thread mahongbin

KYLIN-1355 Fix package version bug and add script to build a java-hotfix 
package, instead of a full package

Changes:
* Add build/script/package_libs.sh  to build library jars only tarball
* Fix version parsing bug in build/script/package.sh

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5bcb78d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5bcb78d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5bcb78d0

Branch: refs/heads/2.0-rc
Commit: 5bcb78d066bc02b449d6c946396d0e74d43d1f41
Parents: 7c62e36
Author: Hao Chen 
Authored: Wed Jan 27 17:44:11 2016 +0800
Committer: honma 
Committed: Wed Jan 27 21:52:55 2016 +0800

--
 build/script/package.sh  |   2 +-
 build/script/package_libs.sh | 100 ++
 2 files changed, 101 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5bcb78d0/build/script/package.sh
--
diff --git a/build/script/package.sh b/build/script/package.sh
index a505501..53c2908 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -50,7 +50,7 @@ fi
 
 dir=$(dirname ${0})
 cd ${dir}/../..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate 
-Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+'`
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate 
-Dexpression=project.version | grep -v '\['`
 echo "kylin version: ${version}"
 export version
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5bcb78d0/build/script/package_libs.sh
--
diff --git a/build/script/package_libs.sh b/build/script/package_libs.sh
new file mode 100755
index 000..71c1e8c
--- /dev/null
+++ b/build/script/package_libs.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo "Checking maven..."
+
+if [ -z "$(command -v mvn)" ]
+then
+echo "Please install maven first so that Kylin packaging can proceed"
+exit 1
+else
+echo "maven check passed"
+fi
+
+echo "Checking git..."
+
+if [ -z "$(command -v git)" ]
+then
+echo "Please install git first so that Kylin packaging can proceed"
+exit 1
+else
+echo "git check passed"
+fi
+
+dir=$(dirname ${0})
+cd ${dir}/../..
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate 
-Dexpression=project.version | grep -v '\['`
+
+if [ "$version" == "" ];then
+   echo "Failed to identify kylin version (current: `pwd`)"
+   exit 1
+fi
+
+echo "kylin version: ${version}"
+export version
+
+#commit id
+cat << EOF > build/commit_SHA1
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+EOF
+git rev-parse HEAD >> build/commit_SHA1
+
+echo "package libraries"
+mvn clean install -DskipTests   || { exit 1; }
+
+echo "copy libraries"
+rm -rf build/lib
+mkdir build/lib
+cp assembly/target/kylin-assembly-${version}-job.jar 
build/lib/kylin-job-${version}.jar
+cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar 
build/lib/kylin-coprocessor-${version}.jar
+cp jdbc/target/kylin-jdbc-${version}.jar build/lib/kylin-jdbc-${version}.jar
+# Copied file becomes 000 for some env (e.g.

[1/3] kylin git commit: KYLIN-1363 Fix unary operator expected bug

2016-01-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 95970d5e1 -> 5bcb78d06


KYLIN-1363 Fix unary operator expected bug

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2f523ca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2f523ca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2f523ca

Branch: refs/heads/2.0-rc
Commit: c2f523ca407dc2dcd02015ed3f5b559ffe5b969a
Parents: 95970d5
Author: Hao Chen 
Authored: Sun Jan 24 14:28:49 2016 +0800
Committer: honma 
Committed: Wed Jan 27 21:51:05 2016 +0800

--
 build/bin/kylin.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2f523ca/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 77a17a8..107ca73 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -22,7 +22,7 @@ source ${dir}/check-env.sh
 mkdir -p ${KYLIN_HOME}/logs
 
 # start command
-if [ $1 == "start" ]
+if [ "$1" == "start" ]
 then
 if [ -f "${KYLIN_HOME}/pid" ]
 then
@@ -91,7 +91,7 @@ then
 exit 0
 
 # stop command
-elif [ $1 == "stop" ]
+elif [ "$1" == "stop" ]
 then
 if [ -f "${KYLIN_HOME}/pid" ]
 then

[2/3] kylin git commit: Fix more unary operator expected bugs on v2.x

2016-01-27 Thread mahongbin

Fix more unary operator expected bugs on v2.x

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c62e365
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c62e365
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c62e365

Branch: refs/heads/2.0-rc
Commit: 7c62e3652d2b93a58e47c7fc629aeb184a00cc03
Parents: c2f523c
Author: Hao Chen 
Authored: Wed Jan 27 17:58:48 2016 +0800
Committer: honma 
Committed: Wed Jan 27 21:51:12 2016 +0800

--
 build/bin/kylin.sh | 10 +-
 build/bin/metastore.sh |  8 
 2 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7c62e365/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 107ca73..f52f923 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -113,14 +113,14 @@ then
 fi
 
 # streaming command
-elif [ $1 == "streaming" ]
+elif [ "$1" == "streaming" ]
 then
 if [ $# -lt 4 ]
 then
 echo "invalid input args $@"
 exit -1
 fi
-if [ $2 == "start" ]
+if [ "$2" == "start" ]
 then
 
 #retrive $hive_dependency and $hbase_dependency
@@ -142,7 +142,7 @@ then
 org.apache.kylin.engine.streaming.cli.StreamingCLI $@ > 
${KYLIN_HOME}/logs/streaming_$3_$4.log 2>&1 & echo $! > 
${KYLIN_HOME}/logs/$3_$4 &
 echo "streaming started name: $3 id: $4"
 exit 0
-elif [ $2 == "stop" ]
+elif [ "$2" == "stop" ]
 then
 if [ ! -f "${KYLIN_HOME}/$3_$4" ]
 then
@@ -165,7 +165,7 @@ then
 fi
 
 # monitor command
-elif [ $1 == "monitor" ]
+elif [ "$1" == "monitor" ]
 then
 echo "monitor job"
 
@@ -189,7 +189,7 @@ then
 exit 0
 
 # tool command
-elif [[ $1 = org.apache.kylin.* ]]
+elif [[ "$1" = org.apache.kylin.* ]]
 then
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh

http://git-wip-us.apache.org/repos/asf/kylin/blob/7c62e365/build/bin/metastore.sh
--
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 8f237aa..05041f9 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -27,7 +27,7 @@
 dir=$(dirname ${0})
 source ${dir}/check-env.sh
 
-if [ $1 == "backup" ]
+if [ "$1" == "backup" ]
 then
 
 mkdir -p ${KYLIN_HOME}/meta_backups
@@ -40,19 +40,19 @@ then
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool download ${_file}
 echo "metadata store backed up to ${_file}"
 
-elif [ $1 == "restore" ]
+elif [ "$1" == "restore" ]
 then
 
 _file=$2
 echo "Starting restoring $_file"
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool upload $_file
 
-elif [ $1 == "reset" ]
+elif [ "$1" == "reset" ]
 then
 
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool  reset
 
-elif [ $1 == "clean" ]
+elif [ "$1" == "clean" ]
 then
 
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.engine.mr.steps.MetadataCleanupJob "${@:2}"

[1/2] kylin git commit: Fix more unary operator expected bugs on v1.x

2016-01-27 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 4a5d05992 -> d6974474a


Fix more unary operator expected bugs on v1.x

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d6974474
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d6974474
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d6974474

Branch: refs/heads/1.x-staging
Commit: d6974474aaa827793981f79df14f2a6ce8612733
Parents: 4e18dfe
Author: Hao Chen 
Authored: Wed Jan 27 18:00:48 2016 +0800
Committer: honma 
Committed: Wed Jan 27 21:39:14 2016 +0800

--
 bin/kylin.sh | 2 +-
 bin/metastore.sh | 8 
 2 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d6974474/bin/kylin.sh
--
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 5bf3eab..6b0a495 100755
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -115,7 +115,7 @@ then
 fi
 
 # tool command
-elif [[ $1 = org.apache.kylin.* ]]
+elif [[ "$1" = org.apache.kylin.* ]]
 then
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh

http://git-wip-us.apache.org/repos/asf/kylin/blob/d6974474/bin/metastore.sh
--
diff --git a/bin/metastore.sh b/bin/metastore.sh
index 39593d4..5d4ef40 100755
--- a/bin/metastore.sh
+++ b/bin/metastore.sh
@@ -28,7 +28,7 @@
 dir=$(dirname ${0})
 source ${dir}/check-env.sh
 
-if [ $1 == "backup" ]
+if [ "$1" == "backup" ]
 then
 
 mkdir -p ${KYLIN_HOME}/meta_backups
@@ -41,19 +41,19 @@ then
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool download ${_file}
 echo "metadata store backed up to ${_file}"
 
-elif [ $1 == "restore" ]
+elif [ "$1" == "restore" ]
 then
 
 _file=$2
 echo "Starting restoring $_file"
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool upload $_file
 
-elif [ $1 == "reset" ]
+elif [ "$1" == "reset" ]
 then
 
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.common.persistence.ResourceTool  reset
 
-elif [ $1 == "clean" ]
+elif [ "$1" == "clean" ]
 then
 
 ${KYLIN_HOME}/bin/kylin.sh 
org.apache.kylin.job.hadoop.cube.MetadataCleanupJob "${@:2}"

[2/2] kylin git commit: KYLIN-1363 Fix unary operator expected bug

2016-01-27 Thread mahongbin

KYLIN-1363 Fix unary operator expected bug

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4e18dfe2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4e18dfe2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4e18dfe2

Branch: refs/heads/1.x-staging
Commit: 4e18dfe2055ca0c23a3fbff4d7e3175a48e743be
Parents: 4a5d059
Author: Hao Chen 
Authored: Sun Jan 24 14:16:58 2016 +0800
Committer: honma 
Committed: Wed Jan 27 21:39:14 2016 +0800

--
 bin/kylin.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4e18dfe2/bin/kylin.sh
--
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 8ca3c91..5bf3eab 100755
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -22,7 +22,7 @@ source ${dir}/check-env.sh
 mkdir -p ${KYLIN_HOME}/logs
 
 # start command
-if [ $1 == "start" ]
+if [ "$1" == "start" ]
 then
 
 if [ -f "${KYLIN_HOME}/pid" ]
@@ -93,7 +93,7 @@ then
 exit 0
 
 # stop command
-elif [ $1 == "stop" ]
+elif [ "$1" == "stop" ]
 then
 if [ -f "${KYLIN_HOME}/pid" ]
 then

kylin git commit: KYLIN-1356 increase memory settings for failsafe

2016-01-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5511f2609 -> cd5e21b4c


KYLIN-1356 increase memory settings for failsafe


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd5e21b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd5e21b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd5e21b4

Branch: refs/heads/2.x-staging
Commit: cd5e21b4c077a0df5714a427c1cbd34b54a14d13
Parents: 5511f26
Author: honma 
Authored: Wed Jan 27 12:45:34 2016 +0800
Committer: honma 
Committed: Wed Jan 27 12:45:34 2016 +0800

--
 kylin-it/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd5e21b4/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 3f38d16..6cb44a5 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -282,6 +282,7 @@
 true
 
 
+-Xms1G -Xmx4G -XX:PermSize=128M 
-XX:MaxPermSize=512M

[03/17] kylin git commit: minor, update version in pom

2016-01-26 Thread mahongbin

minor, update version in pom


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0d29d63e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0d29d63e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0d29d63e

Branch: refs/heads/2.x-staging
Commit: 0d29d63e7415f69182334bdb5dbea07f519a4829
Parents: 30c9fd6
Author: honma 
Authored: Mon Jan 25 17:42:24 2016 +0800
Committer: honma 
Committed: Tue Jan 26 21:40:48 2016 +0800

--
 assembly/pom.xml |   3 +-
 atopcalcite/pom.xml  |   2 +-
 core-common/pom.xml  |   3 +-
 core-cube/pom.xml|   3 +-
 core-dictionary/pom.xml  |   3 +-
 core-job/pom.xml |   3 +-
 core-metadata/pom.xml|   3 +-
 core-storage/pom.xml |   3 +-
 engine-mr/pom.xml|   3 +-
 engine-spark/pom.xml |   3 +-
 engine-streaming/pom.xml |   3 +-
 invertedindex/pom.xml|   3 +-
 jdbc/pom.xml | 144 +-
 pom.xml  |   3 +-
 query/pom.xml|   3 +-
 server/pom.xml   |   3 +-
 source-hive/pom.xml  |   3 +-
 source-kafka/pom.xml |   3 +-
 storage-hbase/pom.xml|   3 +-
 19 files changed, 108 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index cd7fc2b..999226a 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -5,7 +5,8 @@
 
 kylin
 org.apache.kylin
-2.0-incubating-SNAPSHOT
+2.1-SNAPSHOT
+
 
 4.0.0
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 45cca31..9c296d0 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.1-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 8c7edb1..28566d6 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -28,7 +28,8 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.1-SNAPSHOT
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 58c1a14..3757a8b 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,8 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.1-SNAPSHOT
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 2ea09fb..4add5e8 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,8 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.1-SNAPSHOT
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 97c8f2e..32f149e 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,8 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.1-SNAPSHOT
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 8abf02e..f0bbd40 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -27,7 +27,8 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.1-SNAPSHOT
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index ad440da..2f6fbbb 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -27,7 +27,8 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.1-SNAPSHOT
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0d29d63e/engine-mr/pom.xml
--
diff --git

[16/17] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
 
b/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
deleted file mode 100644
index 23780cf..000
--- 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ITHiveSourceTableLoaderTest extends HBaseMetadataTestCase {
-
-@Before
-public void setup() throws Exception {
-super.createTestMetadata();
-}
-
-@After
-public void after() throws Exception {
-super.cleanupTestMetadata();
-}
-
-@Test
-public void test() throws IOException {
-KylinConfig config = getTestConfig();
-String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", 
"EDW.TEST_CAL_DT" };
-Set loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, 
config);
-
-assertTrue(loaded.size() == toLoad.length);
-for (String str : toLoad)
-assertTrue(loaded.contains(str));
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
 
b/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
deleted file mode 100644
index 57c0be3..000
--- 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import java.io.IOException;
-
-import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This test case need the hive runtime; Please run it with sandbox;
- * @author shaoshi
- *
- * It is in the exclude list of default profile in pom.xml
- */
-public class ITHiveTableReaderTest extends HBaseMetadataTestCase {
-
-@Test
-public void test() throws IOException {
-HiveTableReader reader = new HiveTableReader("default", 
"test_kylin_fact");
-int rowNumber = 0;
-while (reader.next()) {
-String[] row = reader.getRow();
-Assert.assertEquals(9, row.length);
-//System.out.println(ArrayUtils.toString(row));
-rowNumber++;
-}
-
-reader.close();
-Assert.assertEquals(1, rowNumber);
-}
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/assembly/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
--
diff --git

[06/17] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/query/src/test/resources/query/sql_distinct_precisely/query07.sql
--
diff --git a/query/src/test/resources/query/sql_distinct_precisely/query07.sql 
b/query/src/test/resources/query/sql_distinct_precisely/query07.sql
deleted file mode 100644
index 41252c4..000
--- a/query/src/test/resources/query/sql_distinct_precisely/query07.sql
+++ /dev/null
@@ -1,24 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select lstg_format_name,
- sum(price) as GMV,
- count(1) as TRANS_CNT,
- count(distinct leaf_categ_id) as LEAF_CATEG_CNT
- from test_kylin_fact
- group by lstg_format_name

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/query/src/test/resources/query/sql_dynamic/query01.dat
--
diff --git a/query/src/test/resources/query/sql_dynamic/query01.dat 
b/query/src/test/resources/query/sql_dynamic/query01.dat
deleted file mode 100644
index a72c741..000
--- a/query/src/test/resources/query/sql_dynamic/query01.dat
+++ /dev/null
@@ -1,2 +0,0 @@
-FP-GTC
-Collectibles

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/query/src/test/resources/query/sql_dynamic/query01.sql
--
diff --git a/query/src/test/resources/query/sql_dynamic/query01.sql 
b/query/src/test/resources/query/sql_dynamic/query01.sql
deleted file mode 100644
index d777fdf..000
--- a/query/src/test/resources/query/sql_dynamic/query01.sql
+++ /dev/null
@@ -1,28 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, 
test_category_groupings.meta_categ_name, sum(test_kylin_fact.price) as gmv, 
count(*) as trans_cnt 
- from test_kylin_fact 
- inner JOIN edw.test_cal_dt as test_cal_dt 
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
- inner JOIN test_category_groupings 
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id 
- where test_kylin_fact.lstg_format_name = ? 
- and test_category_groupings.meta_categ_name = ? 
- and test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-10-01' 
- group by test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, 
test_category_groupings.meta_categ_name 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/query/src/test/resources/query/sql_dynamic/query02.dat
--
diff --git a/query/src/test/resources/query/sql_dynamic/query02.dat 
b/query/src/test/resources/query/sql_dynamic/query02.dat
deleted file mode 100644
index a72c741..000
--- a/query/src/test/resources/query/sql_dynamic/query02.dat
+++ /dev/null
@@ -1,2 +0,0 @@
-FP-GTC
-Collectibles

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/query/src/test/resources/query/sql_dynamic/query02.sql
--
diff --git a/query/src/test/resources/query/sql_dynamic/query02.sql 
b/query/src/test/resources/query/sql_dynamic/query02.sql
deleted file mode 100644
index eaeccdd..000
--- a/query/src/test/resources/query/sql_dynamic/query02.sql
+++ /dev/null
@@ -1,30 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or

[09/17] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
--
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java 
b/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
deleted file mode 100644
index 680acee..000
--- a/query/src/test/java/org/apache/kylin/query/test/KylinTestBase.java
+++ /dev/null
@@ -1,506 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.test;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.sql.Types;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.LogManager;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.dbunit.Assertion;
-import org.dbunit.database.DatabaseConfig;
-import org.dbunit.database.DatabaseConnection;
-import org.dbunit.database.IDatabaseConnection;
-import org.dbunit.dataset.DataSetException;
-import org.dbunit.dataset.ITable;
-import org.dbunit.dataset.SortedTable;
-import org.dbunit.dataset.datatype.DataType;
-import org.dbunit.dataset.datatype.DataTypeException;
-import org.dbunit.ext.h2.H2Connection;
-import org.dbunit.ext.h2.H2DataTypeFactory;
-import org.junit.Assert;
-
-import com.google.common.io.Files;
-
-/**
- */
-public class KylinTestBase {
-
-// Hack for the different constant integer type between optiq (INTEGER) and
-// h2 (BIGINT)
-public static class TestH2DataTypeFactory extends H2DataTypeFactory {
-@Override
-public DataType createDataType(int sqlType, String sqlTypeName, String 
tableName, String columnName) throws DataTypeException {
-
-if ((columnName.startsWith("COL") || columnName.startsWith("col")) 
&& sqlType == Types.BIGINT) {
-return DataType.INTEGER;
-}
-return super.createDataType(sqlType, sqlTypeName);
-}
-}
-
-protected static final String resultTableName = "query result of ";
-protected static KylinConfig config = null;
-protected static Connection cubeConnection = null;
-protected static Connection h2Connection = null;
-protected static String joinType = "default";
-protected static int h2InstanceCount = 0;
-
-protected static int compQueryCount = 0;
-protected static ArrayList zeroResultQueries = new 
ArrayList();
-
-protected static void closeConnection(Connection connection) {
-if (connection != null) {
-try {
-connection.close();
-} catch (SQLException e) {
-e.printStackTrace();
-}
-}
-}
-
-/**
- * @param folder
- * @param fileType
- *specify the interested file type by file extension
- * @return
- */
-protected static List getFilesFromFolder(final File folder, final 
String fileType) {
-List files = new ArrayList();
-for (final File fileEntry : folder.listFiles()) {
-if 
(fileEntry.getName().toLowerCase().endsWith(fileType.toLowerCase())) {
-files.add(fileEntry);
-}
-}
-return files;
-}
-
-protected static void getFilesFromFolderR(final String directoryStr, 
List files, final String fileType) {
-File folder = new File(directoryStr);
-for (final File fileEntry : folder.listFiles()) {
-if (fileEntry.isDirectory()) {
-getFilesFromFolderR(fileEntry.getAbsolutePath(), files, 
fileType);
-} else if (fileEntry.isFile()) {
-if

[10/17] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/kylin-it/src/test/resources/query/sql_tableau/query21.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query21.sql 
b/kylin-it/src/test/resources/query/sql_tableau/query21.sql
new file mode 100644
index 000..3e3208b
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query21.sql
@@ -0,0 +1,27 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+ select test_kylin_fact.lstg_format_name, 
test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
+ , count(*) as TRANS_CNT 
+ from test_kylin_fact 
+ inner JOIN EDW.TEST_CAL_DT AS TEST_CAL_DT 
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
+ where test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01' 
+ group by test_kylin_fact.lstg_format_name, test_cal_dt.week_beg_dt 
+ having sum(price)>500
+ limit 1

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/kylin-it/src/test/resources/query/sql_tableau/query22.sql.disabled
--
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query22.sql.disabled 
b/kylin-it/src/test/resources/query/sql_tableau/query22.sql.disabled
new file mode 100644
index 000..17ec731
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query22.sql.disabled
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select fact.cal_dt, sum(fact.price) from test_kylin_fact fact 
+left join test_cal_dt cal 
+   on fact.cal_dt=cal.cal_dt
+inner join
+(
+   select test_kylin_fact.cal_dt, count(1) from test_kylin_fact left join 
test_cal_dt 
+   on test_kylin_fact.cal_dt=test_cal_dt.cal_dt group by 
test_kylin_fact.cal_dt order by 2 desc limit 2
+) cal_2 
+   on fact.cal_dt = cal_2.cal_dt 
+group by fact.cal_dt

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/kylin-it/src/test/resources/query/sql_tableau/query23.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query23.sql 
b/kylin-it/src/test/resources/query/sql_tableau/query23.sql
new file mode 100644
index 000..10430ab
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query23.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select fact.cal_dt, sum(fact.price) from test_kylin_fact fact 
+left join EDW.TEST_CAL_DT cal on fact.cal_dt=cal.cal_dt
+where cal.cal_dt  = date '2012-05-17' or cal.cal_dt  = date '2013-05-17'
+group by fact.cal_dt

[11/17] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/kylin-it/src/test/resources/query/sql_invalid/query30_invalid_SQL.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_invalid/query30_invalid_SQL.sql 
b/kylin-it/src/test/resources/query/sql_invalid/query30_invalid_SQL.sql
new file mode 100644
index 000..229d198
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_invalid/query30_invalid_SQL.sql
@@ -0,0 +1,54 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT 
+ test_cal_dt.week_beg_dt_test 
+ ,test_cal_dt.retail_year 
+ ,test_cal_dt.rtl_month_of_rtl_year_id 
+ ,test_cal_dt.retail_week 
+ ,test_category_groupings.meta_categ_name 
+ ,test_category_groupings.categ_lvl2_name 
+ ,test_category_groupings.categ_lvl3_name 
+ ,test_kylin_fact.lstg_format_name 
+ ,test_sites.site_name 
+ ,test_seller_type_dim.seller_type_desc 
+ ,sum(test_kylin_fact.price) as gmv 
+ , count(*) as trans_cnt 
+ FROM test_kylin_fact 
+ inner JOIN test_cal_dt 
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
+ inner JOIN test_category_groupings 
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id 
+ inner JOIN test_sites 
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id 
+ inner JOIN test_seller_type_dim 
+ ON test_kylin_fact.slr_segment_cd = test_seller_type_dim.seller_type_cd 
+ where test_cal_dt.retail_year='2013' 
+ and retail_week in(1,2,3,4,5,6,7,7,7) 
+ and (test_category_groupings.meta_categ_name='Collectibles' or 
test_category_groupings.categ_lvl3_name='Dresses') 
+ and test_sites.site_name='Ebay' 
+ and test_cal_dt.retail_year not in ('2014') 
+ group by test_cal_dt.week_beg_dt 
+ ,test_cal_dt.retail_year 
+ ,test_cal_dt.rtl_month_of_rtl_year_id 
+ ,test_cal_dt.retail_week 
+ ,test_category_groupings.meta_categ_name 
+ ,test_category_groupings.categ_lvl2_name 
+ ,test_category_groupings.categ_lvl3_name 
+ ,test_kylin_fact.lstg_format_name 
+ ,test_sites.site_name 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1428bbc4/kylin-it/src/test/resources/query/sql_invalid/query31_invalid_SQL.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_invalid/query31_invalid_SQL.sql 
b/kylin-it/src/test/resources/query/sql_invalid/query31_invalid_SQL.sql
new file mode 100644
index 000..8fac5a6
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_invalid/query31_invalid_SQL.sql
@@ -0,0 +1,55 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT 
+ test_cal_dt.week_beg_dt_test 
+ ,test_cal_dt.retail_year 
+ ,test_cal_dt.rtl_month_of_rtl_year_id 
+ ,test_cal_dt.retail_week 
+ ,test_category_groupings.meta_categ_name 
+ ,test_category_groupings.categ_lvl2_name 
+ ,test_category_groupings.categ_lvl3_name 
+ ,test_kylin_fact.lstg_format_name 
+ ,test_sites.site_name 
+ ,test_seller_type_dim.seller_type_desc 
+ ,sum(test_kylin_fact.price_amt) as gmv 
+ , count(*) as trans_cnt 
+ FROM test_kylin_fact 
+ inner JOIN test_cal_dt 
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
+ inner JOIN test_category_groupings 
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id 
+ inner JOIN test_sites 
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id 
+ inner

[02/17] kylin git commit: minor, avoid verbose logging in Serializer

2016-01-26 Thread mahongbin

minor, avoid verbose logging in Serializer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2673969
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2673969
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2673969

Branch: refs/heads/2.x-staging
Commit: d2673969fb40436d1c55f359b2b709c55cfab8da
Parents: 0d29d63
Author: honma 
Authored: Tue Jan 26 09:54:57 2016 +0800
Committer: honma 
Committed: Tue Jan 26 21:40:48 2016 +0800

--
 .../java/org/apache/kylin/cube/gridtable/FixLenSerializer.java  | 5 -
 .../apache/kylin/metadata/datatype/BigDecimalSerializer.java| 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d2673969/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java
index 2b70e79..24c4a19 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/FixLenSerializer.java
@@ -36,6 +36,7 @@ public class FixLenSerializer extends DataTypeSerializer {
 private ThreadLocal current = new ThreadLocal();
 
 private int fixLen;
+transient int avoidVerbose = 0;
 
 FixLenSerializer(int fixLen) {
 this.fixLen = fixLen;
@@ -59,7 +60,9 @@ public class FixLenSerializer extends DataTypeSerializer {
 } else {
 byte[] bytes = Bytes.toBytes(value.toString());
 if (bytes.length > fixLen) {
-logger.warn("Expect at most " + fixLen + " bytes, but got " + 
bytes.length + ", will truncate, value string: " + value.toString());
+if (avoidVerbose++ % 1 == 0) {
+logger.warn("Expect at most " + fixLen + " bytes, but got 
" + bytes.length + ", will truncate, value string: " + value.toString() + " 
times:" + avoidVerbose);
+}
 }
 out.put(bytes, 0, Math.min(bytes.length, fixLen));
 for (int i = bytes.length; i < fixLen; i++) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/d2673969/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
index 25b4e07..64968b8 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
@@ -48,8 +48,8 @@ public class BigDecimalSerializer extends 
DataTypeSerializer {
 @Override
 public void serialize(BigDecimal value, ByteBuffer out) {
 if (value.scale() > type.getScale()) {
-if (avoidVerbose % 1 == 0) {
-logger.warn("value's scale has exceeded the " + 
type.getScale() + ", cut it off, to ensure encoded value do not exceed 
maxLength " + maxLength + " times:" + (avoidVerbose++));
+if (avoidVerbose++ % 1 == 0) {
+logger.warn("value's scale has exceeded the " + 
type.getScale() + ", cut it off, to ensure encoded value do not exceed 
maxLength " + maxLength + " times:" + (avoidVerbose));
 }
 value = value.setScale(type.getScale(), 
BigDecimal.ROUND_HALF_EVEN);
 }

kylin git commit: KYLIN-1356 fix bug

2016-01-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 0b676d5bf -> 5511f2609


KYLIN-1356 fix bug


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5511f260
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5511f260
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5511f260

Branch: refs/heads/2.x-staging
Commit: 5511f26095982a4e3b021414cd21bba3378772e4
Parents: 0b676d5
Author: honma 
Authored: Wed Jan 27 11:02:36 2016 +0800
Committer: honma 
Committed: Wed Jan 27 11:02:56 2016 +0800

--
 kylin-it/pom.xml | 1 -
 pom.xml  | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5511f260/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 3fc867d..3f38d16 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -31,7 +31,6 @@
 kylin-it
 Kylin:IT
 
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5511f260/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ba87d90..21fd8aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -168,7 +168,7 @@
 
scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git
 
scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git
 https://git-wip-us.apache.org/repos/asf/kylin.git
-apache-kylin-2.0-incubating
+apache-kylin-2.1

kylin git commit: minor, update version in pom

2016-01-25 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 5d10249f1 -> 3c3615bbf


minor, update version in pom


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3c3615bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3c3615bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3c3615bb

Branch: refs/heads/2.0-rc
Commit: 3c3615bbf0d2512b6f3ff815789efdc4a8169785
Parents: 5d10249
Author: honma 
Authored: Mon Jan 25 17:38:25 2016 +0800
Committer: honma 
Committed: Mon Jan 25 17:38:25 2016 +0800

--
 assembly/pom.xml | 2 +-
 atopcalcite/pom.xml  | 2 +-
 build/bin/upgrade_v2.sh  | 2 +-
 core-common/pom.xml  | 2 +-
 core-cube/pom.xml| 2 +-
 core-dictionary/pom.xml  | 2 +-
 core-job/pom.xml | 2 +-
 core-metadata/pom.xml| 2 +-
 core-storage/pom.xml | 2 +-
 engine-mr/pom.xml| 2 +-
 engine-spark/pom.xml | 3 ++-
 engine-streaming/pom.xml | 2 +-
 invertedindex/pom.xml| 3 ++-
 jdbc/pom.xml | 3 ++-
 kylin-it/pom.xml | 2 +-
 pom.xml  | 2 +-
 query/pom.xml| 3 ++-
 server/pom.xml   | 3 ++-
 source-hive/pom.xml  | 3 ++-
 source-kafka/pom.xml | 3 ++-
 storage-hbase/pom.xml| 3 ++-
 21 files changed, 29 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3c3615bb/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index f7ad8fa..2dfa566 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -5,7 +5,7 @@
 
 kylin
 org.apache.kylin
-2.0-incubating-SNAPSHOT
+2.0-SNAPSHOT
 
 4.0.0
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3c3615bb/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 45cca31..de2ae2d 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.0-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3c3615bb/build/bin/upgrade_v2.sh
--
diff --git a/build/bin/upgrade_v2.sh b/build/bin/upgrade_v2.sh
index 605479f..28761d1 100755
--- a/build/bin/upgrade_v2.sh
+++ b/build/bin/upgrade_v2.sh
@@ -21,7 +21,7 @@ dir=$(dirname ${0})
 source ${dir}/check-env.sh
 
 echo "==Deploy coprocessor==="
-$KYLIN_HOME/bin/kylin.sh 
org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI 
$KYLIN_HOME/lib/kylin-coprocessor-2.0-incubating-SNAPSHOT.jar all
+$KYLIN_HOME/bin/kylin.sh 
org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI 
$KYLIN_HOME/lib/kylin-coprocessor-2.0-SNAPSHOT.jar all
 
 echo "=Upgrade Cube metadata="
 $KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.v2.CubeMetadataUpgradeV2

http://git-wip-us.apache.org/repos/asf/kylin/blob/3c3615bb/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 8c7edb1..6e5553e 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -28,7 +28,7 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.0-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3c3615bb/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 58c1a14..ecac276 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.0-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3c3615bb/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 2ea09fb..e54f5f3 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.0-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3c3615bb/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 97c8f2e..9d70939 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-2.0-incubating-SNAPSHOT
+2.0-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/3c3615bb/core-metadata/pom.xml

[10/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/h2/query09.sql
--
diff --git a/kylin-it/src/test/resources/query/h2/query09.sql 
b/kylin-it/src/test/resources/query/h2/query09.sql
new file mode 100644
index 000..e6e5576
--- /dev/null
+++ b/kylin-it/src/test/resources/query/h2/query09.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) from (select test_cal_dt.week_beg_dt 
+ from test_kylin_fact 
+ inner JOIN edw.test_cal_dt as test_cal_dt 
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
+ group by test_cal_dt.week_beg_dt) t 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/h2/query10.sql
--
diff --git a/kylin-it/src/test/resources/query/h2/query10.sql 
b/kylin-it/src/test/resources/query/h2/query10.sql
new file mode 100644
index 000..9511363
--- /dev/null
+++ b/kylin-it/src/test/resources/query/h2/query10.sql
@@ -0,0 +1,25 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_cal_dt.week_beg_dt 
+ from test_kylin_fact 
+ inner JOIN edw.test_cal_dt as test_cal_dt 
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
+ where test_kylin_fact.lstg_format_name='FP-GTC' 
+ and test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01' 
+ group by test_cal_dt.week_beg_dt 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql/query00.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query00.sql 
b/kylin-it/src/test/resources/query/sql/query00.sql
new file mode 100644
index 000..6e4d94b
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query00.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name, sum(price) as GMV 
+ from test_kylin_fact 
+ where lstg_format_name='FP-GTC' 
+ group by lstg_format_name 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query01.sql 
b/kylin-it/src/test/resources/query/sql/query01.sql
new file mode 100644
index 000..9ed1db3
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query01.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this

[13/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

KYLIN-1356 move all it to a seprate module

use exec instead

set hdp.version

fix classnotdef

fix classnot

fix classnotdef

add fastBuildMode

KYLIN-1340

fix license

fix exit

fix jdbc test

fix war

attachclass

fix cache


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59943fee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59943fee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59943fee

Branch: refs/heads/2.0-rc
Commit: 59943fee017fe2aa63c29867acb06fc0d56e5af1
Parents: 5e0e8e0
Author: honma 
Authored: Thu Jan 21 23:32:17 2016 +0800
Committer: honma 
Committed: Sun Jan 24 14:33:10 2016 +0800

--
 assembly/pom.xml|  28 +-
 .../apache/kylin/RealizationRegistryTest.java   |  57 +++
 .../engine/spark/BuildCubeWithSparkTest.java| 151 --
 .../kylin/job/BuildCubeWithEngineTest.java  | 333 
 .../kylin/job/BuildCubeWithStreamTest.java  | 114 -
 .../apache/kylin/job/BuildIIWithEngineTest.java | 253 -
 .../apache/kylin/job/BuildIIWithStreamTest.java | 290 ---
 .../kylin/job/DeployLocalMetaToRemoteTest.java  |   5 +-
 .../hive/ITHiveSourceTableLoaderTest.java   |  55 --
 .../source/hive/ITHiveTableReaderTest.java  |  49 --
 .../source/hive/ITSnapshotManagerTest.java  |  83 ---
 .../kylin/common/persistence/ResourceTool.java  |  11 +
 .../org/apache/kylin/common/util/BasicTest.java |   2 +
 .../common/util/HBaseMetadataTestCase.java  |  60 +++
 .../org/apache/kylin/cube/CubeDescManager.java  |   2 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   4 +-
 .../org/apache/kylin/job/dao/ExecutableDao.java |   4 +-
 .../cache/AbstractCacheFledgedQuery.java|   5 +-
 kylin-it/pom.xml| 426 
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 281 ++
 .../kylin/provision/BuildCubeWithEngine.java| 338 +
 .../kylin/provision/BuildCubeWithSpark.java | 157 ++
 .../kylin/provision/BuildCubeWithStream.java| 100 
 .../kylin/provision/BuildIIWithEngine.java  | 254 ++
 .../kylin/provision/BuildIIWithStream.java  | 298 +++
 .../java/org/apache/kylin/query/H2Database.java | 130 +
 .../apache/kylin/query/ITCombinationTest.java   |  78 +++
 .../org/apache/kylin/query/ITIIQueryTest.java   |  86 
 .../apache/kylin/query/ITKylinQueryTest.java| 238 +
 .../org/apache/kylin/query/KylinTestBase.java   | 507 +++
 .../hive/ITHiveSourceTableLoaderTest.java   |  55 ++
 .../source/hive/ITHiveTableReaderTest.java  |  49 ++
 .../source/hive/ITSnapshotManagerTest.java  |  83 +++
 .../storage/hbase/ITHBaseResourceStoreTest.java | 213 
 .../kylin/storage/hbase/ITHdfsOpsTest.java  |  67 +++
 .../kylin/storage/hbase/ITStorageTest.java  | 163 ++
 .../hbase/ii/ITInvertedIndexHBaseTest.java  | 115 +
 kylin-it/src/test/resources/logging.properties  |   5 +
 .../src/test/resources/query/debug/query78.sql  |  22 +
 .../src/test/resources/query/h2/query07.sql |  21 +
 .../src/test/resources/query/h2/query09.sql |  23 +
 .../src/test/resources/query/h2/query10.sql |  25 +
 .../src/test/resources/query/sql/query00.sql|  22 +
 .../src/test/resources/query/sql/query01.sql|  20 +
 .../src/test/resources/query/sql/query02.sql|  19 +
 .../src/test/resources/query/sql/query03.sql|  20 +
 .../src/test/resources/query/sql/query04.sql|  21 +
 .../src/test/resources/query/sql/query05.sql|  22 +
 .../src/test/resources/query/sql/query06.sql|  23 +
 .../src/test/resources/query/sql/query07.sql|  22 +
 .../src/test/resources/query/sql/query08.sql|  22 +
 .../src/test/resources/query/sql/query09.sql|  27 +
 .../src/test/resources/query/sql/query10.sql|  30 ++
 .../src/test/resources/query/sql/query11.sql|  30 ++
 .../src/test/resources/query/sql/query12.sql|  29 ++
 .../src/test/resources/query/sql/query13.sql|  28 +
 .../src/test/resources/query/sql/query14.sql|  36 ++
 .../src/test/resources/query/sql/query15.sql|  32 ++
 .../src/test/resources/query/sql/query16.sql|  35 ++
 .../src/test/resources/query/sql/query17.sql|  38 ++
 .../src/test/resources/query/sql/query18.sql|  34 ++
 .../src/test/resources/query/sql/query19.sql|  38 ++
 .../src/test/resources/query/sql/query20.sql|  34 ++
 .../src/test/resources/query/sql/query21.sql|  40 ++
 .../src/test/resources/query/sql/query22.sql|  42 ++
 .../src/test/resources/query/sql/query23.sql|  41 ++
 .../src/test/resources/query/sql/query24.sql|  45 ++
 .../src/test/resources/query/sql/query25.sql|  46 ++
 .../src/test/resources/query/sql/query26.sql|  48 ++
 .../src/test/resources/query/sql/query27.sql|  47 ++

[07/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql_invalid/query33_invalid_SQL.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_invalid/query33_invalid_SQL.sql 
b/kylin-it/src/test/resources/query/sql_invalid/query33_invalid_SQL.sql
new file mode 100644
index 000..d18dbcc
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_invalid/query33_invalid_SQL.sql
@@ -0,0 +1,56 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT 
+ test_cal_dt.week_beg_dt_test 
+ ,test_cal_dt.retail_year 
+ ,test_cal_dt.rtl_month_of_rtl_year_id 
+ ,test_cal_dt.retail_week 
+ ,test_category_groupings.meta_categ_name 
+ ,test_category_groupings.categ_lvl2_name 
+ ,test_category_groupings.categ_lvl3_name 
+ ,test_kylin_fact.lstg_format_name 
+ ,test_sites.site_name 
+ ,test_seller_type_dim.seller_type_desc 
+ ,sum(test_kylin_fact.price) as gmv 
+ , count(*) as trans_cnt 
+ FROM test_kylin_fact 
+ inner JOIN test_cal_dt 
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
+ inner JOIN test_category_groupings 
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id 
+ inner JOIN test_sites 
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id 
+ inner JOIN test_seller_type_dim 
+ ON test_kylin_fact.slr_segment_cd = test_seller_type_dim.seller_type_cd 
+ where test_cal_dt.retail_year='2013' 
+ and retail_week in(1,2,3,4,5,6,7,7,7) 
+ and (test_category_groupings.meta_categ_name='Collectibles' or 
test_category_groupings.categ_lvl3_name='Dresses') 
+ and test_sites.site_name='Ebay' 
+ and test_cal_dt.retail_year not in ('2014') 
+ and test_kylin_fact.trans_id=11 
+ group by test_cal_dt.week_beg_dt 
+ ,test_cal_dt.retail_year 
+ ,test_cal_dt.rtl_month_of_rtl_year_id 
+ ,test_cal_dt.retail_week 
+ ,test_category_groupings.meta_categ_name 
+ ,test_category_groupings.categ_lvl2_name 
+ ,test_category_groupings.categ_lvl3_name 
+ ,test_kylin_fact.lstg_format_name 
+ ,test_sites.site_name 
+ ,test_seller_type_dim.seller_type_desc 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql_invalid/query34_invalid_SQL.sql
--
diff --git 
a/kylin-it/src/test/resources/query/sql_invalid/query34_invalid_SQL.sql 
b/kylin-it/src/test/resources/query/sql_invalid/query34_invalid_SQL.sql
new file mode 100644
index 000..4308735
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_invalid/query34_invalid_SQL.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT COUNT(DISTINCT "TableauSQL"."TRANS_CNT") AS "ctd_TRANS_CNT_qk", 
"TableauSQL"."LSTG_FORMAT_NAME" AS "none_LSTG_FORMAT_NAME_nk" 
+ FROM ( select test_kylin_fact.lstg_format_name, sum(price) as GMV, 
count(seller_id) as TRANS_CNT 
+ from test_kylin_fact 
+ group by test_kylin_fact.lstg_format_name 
+ ) "TableauSQL" 
+ GROUP BY "TableauSQL"."LSTG_FORMAT_NAME" 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql_invalid/query_count_distinct_on_dimension.sql.disabled
--
diff --git 
a/kylin-it/src/test/resources/query/sql_invalid/query_count_distinct_on_dimension.sql.disabled

[09/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql/query46.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query46.sql 
b/kylin-it/src/test/resources/query/sql/query46.sql
new file mode 100644
index 000..3bfe9d9
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query46.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT  from test_category_groupings

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql/query47.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query47.sql 
b/kylin-it/src/test/resources/query/sql/query47.sql
new file mode 100644
index 000..cbd2c6d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query47.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT  from edw.test_seller_type_dim

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql/query48.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query48.sql 
b/kylin-it/src/test/resources/query/sql/query48.sql
new file mode 100644
index 000..54ddb31
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query48.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as CNT from edw.test_sites

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql/query49.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query49.sql 
b/kylin-it/src/test/resources/query/sql/query49.sql
new file mode 100644
index 000..a28beb6
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query49.sql
@@ -0,0 +1,36 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR

[11/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
new file mode 100644
index 000..8436687
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
@@ -0,0 +1,298 @@
+/*
+ *
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ *  contributor license agreements. See the NOTICE file distributed with
+ *
+ *  this work for additional information regarding copyright ownership.
+ *
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ *  (the "License"); you may not use this file except in compliance with
+ *
+ *  the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ *  See the License for the specific language governing permissions and
+ *
+ *  limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.provision;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.UUID;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.StreamingBatch;
+import org.apache.kylin.common.util.StreamingMessage;
+import org.apache.kylin.engine.mr.JobBuilderSupport;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.SliceBuilder;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.invertedindex.model.IIRow;
+import org.apache.kylin.job.DeployUtil;
+import org.apache.kylin.job.JoinedFlatTable;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.source.hive.HiveTableReader;
+import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.ii.IICreateHTableJob;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+public class BuildIIWithStream {
+
+private static final Logger logger = 
LoggerFactory.getLogger(BuildIIWithStream.class);
+
+private static final String[] II_NAME = new String[] { 
"test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
+private IIManager iiManager;
+private KylinConfig kylinConfig;
+
+public static void main(String[] args) throws Exception {
+beforeClass();
+BuildIIWithStream buildCubeWithEngine = new BuildIIWithStream();
+buildCubeWithEngine.before();
+buildCubeWithEngine.build();
+logger.info("Build is done");
+afterClass();
+logger.info("Going to exit");
+System.exit(0);
+}
+
+public static void beforeClass() throws Exception {
+logger.info("Adding to classpath: " + new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+}
+
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+}
+
+public void before() throws Exception {
+

[05/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/query/src/test/resources/query/sql/query02.sql
--
diff --git a/query/src/test/resources/query/sql/query02.sql 
b/query/src/test/resources/query/sql/query02.sql
deleted file mode 100644
index 8d27c93..000
--- a/query/src/test/resources/query/sql/query02.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/query/src/test/resources/query/sql/query03.sql
--
diff --git a/query/src/test/resources/query/sql/query03.sql 
b/query/src/test/resources/query/sql/query03.sql
deleted file mode 100644
index a605594..000
--- a/query/src/test/resources/query/sql/query03.sql
+++ /dev/null
@@ -1,20 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select test_kylin_fact.lstg_format_name, sum(price) as GMV, count(*) as 
TRANS_CNT from test_kylin_fact 
- group by test_kylin_fact.lstg_format_name 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/query/src/test/resources/query/sql/query04.sql
--
diff --git a/query/src/test/resources/query/sql/query04.sql 
b/query/src/test/resources/query/sql/query04.sql
deleted file mode 100644
index a3672f5..000
--- a/query/src/test/resources/query/sql/query04.sql
+++ /dev/null
@@ -1,21 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select test_kylin_fact.lstg_format_name,sum(test_kylin_fact.price) as GMV 
- , count(*) as TRANS_CNT from test_kylin_fact 
- group by test_kylin_fact.lstg_format_name having sum(price)>5000 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/query/src/test/resources/query/sql/query05.sql
--
diff --git a/query/src/test/resources/query/sql/query05.sql 
b/query/src/test/resources/query/sql/query05.sql
deleted file mode 100644
index 512356a..000
--- a/query/src/test/resources/query/sql/query05.sql
+++ /dev/null
@@ -1,22 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at

[06/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql_tableau/query27.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query27.sql 
b/kylin-it/src/test/resources/query/sql_tableau/query27.sql
new file mode 100644
index 000..beeb168
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query27.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT "TEST_KYLIN_FACT"."CAL_DT", SUM("TEST_KYLIN_FACT"."PRICE") AS 
"sum_PRICE_ok" FROM "TEST_KYLIN_FACT" 
+  INNER JOIN (
+ SELECT COUNT(1) AS "XTableau_join_flag", 
SUM("TEST_KYLIN_FACT"."PRICE") AS "X__alias__A", "TEST_KYLIN_FACT"."CAL_DT" 
AS "none_CAL_DT_ok"   FROM "TEST_KYLIN_FACT" 
+ GROUP BY "TEST_KYLIN_FACT"."CAL_DT"   ORDER BY 2 DESC   LIMIT 7  )
+
+"t0" ON ("TEST_KYLIN_FACT"."CAL_DT" = "t0"."none_CAL_DT_ok") GROUP BY 
"TEST_KYLIN_FACT"."CAL_DT"

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql_tableau/query28.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query28.sql 
b/kylin-it/src/test/resources/query/sql_tableau/query28.sql
new file mode 100644
index 000..48cc342
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query28.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- This query don't result exact same average number as H2 DB
+
+select lstg_format_name, avg(price) as GMV
+ from test_kylin_fact 
+ group by lstg_format_name 

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/kylin-it/src/test/resources/query/sql_tableau/query67.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query67.sql 
b/kylin-it/src/test/resources/query/sql_tableau/query67.sql
new file mode 100644
index 000..25cc110
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query67.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT (CASE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" WHEN 'Auction' THEN '111' 
ELSE '222' END) AS "LSTG_FORMAT_NAME__group_",
+  SUM("TEST_KYLIN_FACT"."PRICE") AS "sum_PRICE_ok"
+FROM "TEST_KYLIN_FACT" "TEST_KYLIN_FACT"
+--group by (CASE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" WHEN 'Auction' THEN 
'111' ELSE '222' END)  ORDER BY 1 ASC
+GROUP BY "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME"  ORDER BY 1 ASC

[12/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
 
b/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
deleted file mode 100644
index 23780cf..000
--- 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ITHiveSourceTableLoaderTest extends HBaseMetadataTestCase {
-
-@Before
-public void setup() throws Exception {
-super.createTestMetadata();
-}
-
-@After
-public void after() throws Exception {
-super.cleanupTestMetadata();
-}
-
-@Test
-public void test() throws IOException {
-KylinConfig config = getTestConfig();
-String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", 
"EDW.TEST_CAL_DT" };
-Set loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, 
config);
-
-assertTrue(loaded.size() == toLoad.length);
-for (String str : toLoad)
-assertTrue(loaded.contains(str));
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
 
b/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
deleted file mode 100644
index 57c0be3..000
--- 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import java.io.IOException;
-
-import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This test case need the hive runtime; Please run it with sandbox;
- * @author shaoshi
- *
- * It is in the exclude list of default profile in pom.xml
- */
-public class ITHiveTableReaderTest extends HBaseMetadataTestCase {
-
-@Test
-public void test() throws IOException {
-HiveTableReader reader = new HiveTableReader("default", 
"test_kylin_fact");
-int rowNumber = 0;
-while (reader.next()) {
-String[] row = reader.getRow();
-Assert.assertEquals(9, row.length);
-//System.out.println(ArrayUtils.toString(row));
-rowNumber++;
-}
-
-reader.close();
-Assert.assertEquals(1, rowNumber);
-}
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/assembly/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
--
diff --git

[02/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-23 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/query/src/test/resources/query/sql_lookup/query03.sql
--
diff --git a/query/src/test/resources/query/sql_lookup/query03.sql 
b/query/src/test/resources/query/sql_lookup/query03.sql
deleted file mode 100644
index f663627..000
--- a/query/src/test/resources/query/sql_lookup/query03.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select SELLER_TYPE_DESC, SELLER_TYPE_CD from edw.test_seller_type_dim

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/query/src/test/resources/query/sql_lookup/query04.sql
--
diff --git a/query/src/test/resources/query/sql_lookup/query04.sql 
b/query/src/test/resources/query/sql_lookup/query04.sql
deleted file mode 100644
index 69a77d7..000
--- a/query/src/test/resources/query/sql_lookup/query04.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select SITE_NAME, SITE_ID from edw.test_sites

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/query/src/test/resources/query/sql_optimize/enable-limit01.sql
--
diff --git a/query/src/test/resources/query/sql_optimize/enable-limit01.sql 
b/query/src/test/resources/query/sql_optimize/enable-limit01.sql
deleted file mode 100644
index 4a62d92..000
--- a/query/src/test/resources/query/sql_optimize/enable-limit01.sql
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select * from test_kylin_fact limit 10

http://git-wip-us.apache.org/repos/asf/kylin/blob/59943fee/query/src/test/resources/query/sql_orderby/query01.sql
--
diff --git a/query/src/test/resources/query/sql_orderby/query01.sql 
b/query/src/test/resources/query/sql_orderby/query01.sql
deleted file mode 100644
index a1d5065..000
--- a/query/src/test/resources/query/sql_orderby/query01.sql
+++ /dev/null
@@ -1,32 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or

[1/3] kylin git commit: KYLIN-1340

2016-01-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1356 5f5bc29d5 -> 16dd4070a


KYLIN-1340


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c7b2adb5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c7b2adb5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c7b2adb5

Branch: refs/heads/KYLIN-1356
Commit: c7b2adb5beaa2697cfa2521a3c817d007b15a930
Parents: 5f5bc29
Author: honma 
Authored: Fri Jan 22 15:31:44 2016 +0800
Committer: honma 
Committed: Fri Jan 22 15:31:44 2016 +0800

--
 .../kylin/common/persistence/ResourceTool.java  |  11 +
 .../org/apache/kylin/cube/CubeDescManager.java  |   2 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   4 +-
 .../org/apache/kylin/job/dao/ExecutableDao.java |   4 +-
 .../storage/hbase/util/CubeMetaExtractor.java   | 284 +++
 5 files changed, 300 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c7b2adb5/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 4a23ba3..8faeadf 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -20,7 +20,10 @@ package org.apache.kylin.common.persistence;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
 
+import com.google.common.base.Function;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.StringUtil;
 
@@ -76,6 +79,14 @@ public class ResourceTool {
 System.out.println("" + result);
 }
 
+public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, 
List paths) throws IOException {
+ResourceStore src = ResourceStore.getStore(srcConfig);
+ResourceStore dst = ResourceStore.getStore(dstConfig);
+for (String path : paths) {
+copyR(src, dst, path);
+}
+}
+
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) 
throws IOException {
 
 ResourceStore src = ResourceStore.getStore(srcConfig);

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7b2adb5/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 53cd00f..424fd80 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -112,7 +112,7 @@ public class CubeDescManager {
 public CubeDesc reloadCubeDescLocal(String name) throws IOException {
 
 // Save Source
-String path = CubeDesc.getCubeDescResourcePath(name);
+String path = CubeDesc.concatResourcePath(name);
 
 // Reload the CubeDesc
 CubeDesc ndesc = loadCubeDesc(path);

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7b2adb5/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index bbc9f90..db05d7e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -267,10 +267,10 @@ public class CubeDesc extends RootPersistentEntity {
 }
 
 public String getResourcePath() {
-return getCubeDescResourcePath(name);
+return concatResourcePath(name);
 }
 
-public static String getCubeDescResourcePath(String descName) {
+public static String concatResourcePath(String descName) {
 return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + 
MetadataConstants.FILE_SURFIX;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7b2adb5/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java 
b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 18e36b4..d479dfe 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++

[2/3] kylin git commit: fix license

2016-01-22 Thread mahongbin

fix license


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1bb69a93
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1bb69a93
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1bb69a93

Branch: refs/heads/KYLIN-1356
Commit: 1bb69a9335512e0cbed3ec8a0c756cdffd3f104d
Parents: c7b2adb
Author: honma 
Authored: Fri Jan 22 16:09:48 2016 +0800
Committer: honma 
Committed: Fri Jan 22 16:09:48 2016 +0800

--
 kylin-it/pom.xml | 37 +++--
 1 file changed, 19 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1bb69a93/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 7112f21..6813f34 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -1,21 +1,22 @@
 
 
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+ http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
 
 http://maven.apache.org/POM/4.0.0;
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
@@ -32,8 +33,8 @@
 
 
 
-
-
+
+

kylin git commit: fix war

2016-01-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1356 1918704d1 -> 53eb5cd94


fix war


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53eb5cd9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53eb5cd9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53eb5cd9

Branch: refs/heads/KYLIN-1356
Commit: 53eb5cd94e6fbddaaaffd59364c8de6b19447a9f
Parents: 1918704
Author: honma 
Authored: Fri Jan 22 19:42:45 2016 +0800
Committer: honma 
Committed: Fri Jan 22 19:42:45 2016 +0800

--
 kylin-it/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/53eb5cd9/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index c8ec8bc..8322a1f 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -47,6 +47,7 @@
 
 org.apache.kylin
 kylin-server
+war
 ${project.parent.version}

kylin git commit: attachclass

2016-01-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1356 53eb5cd94 -> 3e51b9a33


attachclass


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3e51b9a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3e51b9a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3e51b9a3

Branch: refs/heads/KYLIN-1356
Commit: 3e51b9a3368eb2126ef72dc2a08190ed97debea9
Parents: 53eb5cd
Author: honma 
Authored: Fri Jan 22 22:46:35 2016 +0800
Committer: honma 
Committed: Fri Jan 22 22:46:35 2016 +0800

--
 kylin-it/pom.xml |  2 +-
 server/pom.xml   | 19 ++-
 2 files changed, 15 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3e51b9a3/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 8322a1f..a855fa0 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -47,8 +47,8 @@
 
 org.apache.kylin
 kylin-server
-war
 ${project.parent.version}
+classes
 
 
 org.apache.kylin

http://git-wip-us.apache.org/repos/asf/kylin/blob/3e51b9a3/server/pom.xml
--
diff --git a/server/pom.xml b/server/pom.xml
index fb20d49..5e0a813 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -17,7 +17,8 @@
  limitations under the License.
 -->
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 kylin-server
@@ -81,7 +82,7 @@
 test
 ${project.parent.version}
 
-
+
 
 
 org.apache.kylin
@@ -257,7 +258,7 @@
 super-csv
 2.1.0
 
-   
+
 
 
 org.aspectj
@@ -384,7 +385,7 @@
 
 
 
- 
+
 org.apache.hive.hcatalog
 hive-hcatalog-core
 ${hive-hcatalog.version}
@@ -399,7 +400,7 @@
 jsp-api
 
 
-  
+
 
 
 org.apache.tomcat
@@ -468,6 +469,14 @@
 
 
 
+maven-war-plugin
+2.1.1
+
+true
+classes
+
+
+
 maven-compiler-plugin

kylin git commit: KYLIN-1356 use exec-maven-plugin for IT environment provision

2016-01-21 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 6485324a2 -> ab853f9dc


KYLIN-1356 use exec-maven-plugin for IT environment provision


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab853f9d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab853f9d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab853f9d

Branch: refs/heads/2.0-rc
Commit: ab853f9dc987549ac27dd2bb3db1732e3e474281
Parents: 6485324
Author: honma 
Authored: Thu Jan 21 18:20:10 2016 +0800
Committer: honma 
Committed: Thu Jan 21 18:20:19 2016 +0800

--
 .../engine/spark/BuildCubeWithSparkTest.java|   3 +
 .../apache/kylin/job/BuildCubeWithEngine.java   | 333 +++
 .../kylin/job/BuildCubeWithEngineTest.java  | 333 ---
 .../apache/kylin/job/BuildCubeWithStream.java   | 110 ++
 .../kylin/job/BuildCubeWithStreamTest.java  | 114 ---
 .../apache/kylin/job/BuildIIWithEngineTest.java |   7 +-
 .../org/apache/kylin/job/BuildIIWithStream.java | 292 
 .../apache/kylin/job/BuildIIWithStreamTest.java | 290 
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   3 -
 pom.xml |  73 ++--
 10 files changed, 789 insertions(+), 769 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ab853f9d/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
 
b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index f5b9741..f87fa99 100644
--- 
a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -46,10 +46,13 @@ import 
org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 
+//TODO: convert it to a normal class rather than a test case, like in 
BuildCubeWithEngine 
+@Ignore
 public class BuildCubeWithSparkTest {
 
 private CubeManager cubeManager;

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab853f9d/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngine.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngine.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngine.java
new file mode 100644
index 000..2562a8b
--- /dev/null
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngine.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import

[05/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-21 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/query/src/test/resources/query/sql/query12.sql
--
diff --git a/query/src/test/resources/query/sql/query12.sql 
b/query/src/test/resources/query/sql/query12.sql
deleted file mode 100644
index 012ecfc..000
--- a/query/src/test/resources/query/sql/query12.sql
+++ /dev/null
@@ -1,29 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select test_kylin_fact.lstg_format_name, 
test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV 
- , count(*) as TRANS_CNT 
- from test_kylin_fact 
-inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- inner JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id
- where test_cal_dt.week_beg_dt >= DATE '2013-02-10' 
- group by test_kylin_fact.lstg_format_name, test_cal_dt.week_beg_dt 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/query/src/test/resources/query/sql/query13.sql
--
diff --git a/query/src/test/resources/query/sql/query13.sql 
b/query/src/test/resources/query/sql/query13.sql
deleted file mode 100644
index 7d1c481..000
--- a/query/src/test/resources/query/sql/query13.sql
+++ /dev/null
@@ -1,28 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select sum(test_kylin_fact.price) as GMV 
- , count(*) as TRANS_CNT from test_kylin_fact 
-inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- inner JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id
- where test_kylin_fact.lstg_format_name='FP-GTC' 
- and test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01' 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/query/src/test/resources/query/sql/query14.sql
--
diff --git a/query/src/test/resources/query/sql/query14.sql 
b/query/src/test/resources/query/sql/query14.sql
deleted file mode 100644
index 9cca8f5..000
--- a/query/src/test/resources/query/sql/query14.sql
+++ /dev/null
@@ -1,36 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT 
-

[10/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-21 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/kylin-it/src/test/resources/query/sql/query07.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query07.sql 
b/kylin-it/src/test/resources/query/sql/query07.sql
new file mode 100644
index 000..0c27d98
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query07.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.lstg_format_name,sum(test_kylin_fact.price) as GMV 
+ , count(*) as TRANS_CNT from test_kylin_fact 
+ where test_kylin_fact.lstg_format_name='FP-GTC' 
+ group by test_kylin_fact.lstg_format_name having sum(price)>5000 or 
count(*)>20 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/kylin-it/src/test/resources/query/sql/query08.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query08.sql 
b/kylin-it/src/test/resources/query/sql/query08.sql
new file mode 100644
index 000..7a31541
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query08.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.lstg_format_name,sum(test_kylin_fact.price) as GMV 
+ , count(*) as TRANS_CNT from test_kylin_fact 
+ where not (test_kylin_fact.lstg_format_name='FP-GTC') 
+ group by test_kylin_fact.lstg_format_name 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/kylin-it/src/test/resources/query/sql/query09.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query09.sql 
b/kylin-it/src/test/resources/query/sql/query09.sql
new file mode 100644
index 000..d13f8c8
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query09.sql
@@ -0,0 +1,27 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_cal_dt.week_beg_dt, count(*) as TRANS_CNT 
+ from test_kylin_fact 
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ group by test_cal_dt.week_beg_dt 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/kylin-it/src/test/resources/query/sql/query10.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query10.sql

[01/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-21 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1356 [created] 7f08dab55


http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
deleted file mode 100644
index 1546576..000
--- a/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.jdbc;
-
-import java.io.File;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.webapp.WebAppContext;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class ITJDBCDriverTest extends HBaseMetadataTestCase {
-
-private static Server server = null;
-private static SystemPropertiesOverride sysPropsOverride = new 
SystemPropertiesOverride();
-
-@BeforeClass
-public static void beforeClass() throws Exception {
-sysPropsOverride.override("spring.profiles.active", "testing");
-sysPropsOverride.override("catalina.home", "."); // 
resources/log4j.properties ref ${catalina.home}
-staticCreateTestMetadata();
-startJetty();
-}
-
-@AfterClass
-public static void afterClass() throws Exception {
-stopJetty();
-staticCleanupTestMetadata();
-sysPropsOverride.restore();
-}
-
-protected static void stopJetty() throws Exception {
-if (server != null)
-server.stop();
-
-File workFolder = new File("work");
-if (workFolder.isDirectory() && workFolder.exists()) {
-FileUtils.deleteDirectory(workFolder);
-}
-}
-
-protected static void startJetty() throws Exception {
-
-server = new Server(7070);
-
-WebAppContext context = new WebAppContext();
-context.setDescriptor("./src/main/webapp/WEB-INF/web.xml");
-context.setResourceBase("./src/main/webapp");
-context.setContextPath("/kylin");
-context.setParentLoaderPriority(true);
-
-server.setHandler(context);
-
-server.start();
-
-}
-
-protected Connection getConnection() throws Exception {
-
-Driver driver = (Driver) 
Class.forName("org.apache.kylin.jdbc.Driver").newInstance();
-Properties info = new Properties();
-info.put("user", "ADMIN");
-info.put("password", "KYLIN");
-Connection conn = 
driver.connect("jdbc:kylin://localhost:7070/default", info);
-
-return conn;
-}
-
-@Test
-public void testMetadata1() throws Exception {
-
-// check the JDBC API here: 
http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html
-Connection conn = getConnection();
-
-// test getSchemas();
-List schemaList = Lists.newArrayList();
-DatabaseMetaData dbMetadata = conn.getMetaData();
-ResultSet resultSet = dbMetadata.getSchemas();
-while (resultSet.next()) {
-String schema = resultSet.getString("TABLE_SCHEM");
-String catalog = resultSet.getString("TABLE_CATALOG");
-
-System.out.println("Get schema: schema=" + schema + ", catalog=" + 
catalog);
-schemaList.add(schema);
-
-}
-
-resultSet.close();
-Assert.assertTrue(schemaList.contains("DEFAULT"));
-Assert.assertTrue(schemaList.contains("EDW"));
-
-// test getCatalogs();
-resultSet = dbMetadata.getCatalogs();
-
-List catalogList = Lists.newArrayList();
-while (resultSet.next()) {
-String catalog =

[12/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-21 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
 
b/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
deleted file mode 100644
index 57c0be3..000
--- 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import java.io.IOException;
-
-import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This test case need the hive runtime; Please run it with sandbox;
- * @author shaoshi
- *
- * It is in the exclude list of default profile in pom.xml
- */
-public class ITHiveTableReaderTest extends HBaseMetadataTestCase {
-
-@Test
-public void test() throws IOException {
-HiveTableReader reader = new HiveTableReader("default", 
"test_kylin_fact");
-int rowNumber = 0;
-while (reader.next()) {
-String[] row = reader.getRow();
-Assert.assertEquals(9, row.length);
-//System.out.println(ArrayUtils.toString(row));
-rowNumber++;
-}
-
-reader.close();
-Assert.assertEquals(1, rowNumber);
-}
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/assembly/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
 
b/assembly/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
deleted file mode 100644
index 0df632a..000
--- 
a/assembly/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.source.ReadableTable;
-import org.apache.kylin.source.ReadableTable.TableReader;
-import org.apache.kylin.source.SourceFactory;
-import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- * 
- */
-public class ITSnapshotManagerTest extends HBaseMetadataTestCase {
-
-SnapshotManager snapshotMgr;
-
-@Before
-public void setup() throws Exception {
-createTestMetadata();
-snapshotMgr = SnapshotManager.getInstance(getTestConfig());
-}
-
-@After
-public void after() throws Exception {
-cleanupTestMetadata();
-}
-
-@Test
-public void basicTest() throws Exception {
-String tableName = "EDW.TEST_SITES";
-TableDesc tableDesc = 
MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
-ReadableTable hiveTable = SourceFactory.createReadableTable(tableDesc);
-

[13/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-21 Thread mahongbin

KYLIN-1356 move all it to a seprate module


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f08dab5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f08dab5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f08dab5

Branch: refs/heads/KYLIN-1356
Commit: 7f08dab55a94e136c1eda01211765fb11c787810
Parents: ab853f9
Author: honma 
Authored: Thu Jan 21 23:32:17 2016 +0800
Committer: honma 
Committed: Thu Jan 21 23:32:17 2016 +0800

--
 assembly/pom.xml|  28 +-
 .../apache/kylin/RealizationRegistryTest.java   |  57 +++
 .../engine/spark/BuildCubeWithSparkTest.java| 154 --
 .../apache/kylin/job/BuildCubeWithEngine.java   | 333 
 .../apache/kylin/job/BuildCubeWithStream.java   | 110 
 .../apache/kylin/job/BuildIIWithEngineTest.java | 252 -
 .../org/apache/kylin/job/BuildIIWithStream.java | 292 ---
 .../kylin/job/DeployLocalMetaToRemoteTest.java  |   2 +-
 .../hive/ITHiveSourceTableLoaderTest.java   |  55 --
 .../source/hive/ITHiveTableReaderTest.java  |  49 --
 .../source/hive/ITSnapshotManagerTest.java  |  83 ---
 .../org/apache/kylin/common/util/BasicTest.java |   2 +
 .../common/util/HBaseMetadataTestCase.java  |  63 +++
 kylin-it/pom.xml| 367 ++
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 281 ++
 .../kylin/provision/BuildCubeWithEngine.java| 334 
 .../kylin/provision/BuildCubeWithSpark.java | 155 ++
 .../kylin/provision/BuildCubeWithStream.java|  95 
 .../kylin/provision/BuildIIWithEngine.java  | 253 +
 .../kylin/provision/BuildIIWithStream.java  | 294 +++
 .../java/org/apache/kylin/query/H2Database.java | 130 +
 .../apache/kylin/query/ITCombinationTest.java   |  78 +++
 .../org/apache/kylin/query/ITIIQueryTest.java   |  86 
 .../apache/kylin/query/ITKylinQueryTest.java| 238 +
 .../org/apache/kylin/query/KylinTestBase.java   | 507 +++
 .../hive/ITHiveSourceTableLoaderTest.java   |  55 ++
 .../source/hive/ITHiveTableReaderTest.java  |  49 ++
 .../source/hive/ITSnapshotManagerTest.java  |  83 +++
 .../storage/hbase/ITHBaseResourceStoreTest.java | 213 
 .../kylin/storage/hbase/ITHdfsOpsTest.java  |  67 +++
 .../kylin/storage/hbase/ITStorageTest.java  | 163 ++
 .../hbase/ii/ITInvertedIndexHBaseTest.java  | 115 +
 kylin-it/src/test/resources/logging.properties  |   5 +
 .../src/test/resources/query/debug/query78.sql  |  22 +
 .../src/test/resources/query/h2/query07.sql |  21 +
 .../src/test/resources/query/h2/query09.sql |  23 +
 .../src/test/resources/query/h2/query10.sql |  25 +
 .../src/test/resources/query/sql/query00.sql|  22 +
 .../src/test/resources/query/sql/query01.sql|  20 +
 .../src/test/resources/query/sql/query02.sql|  19 +
 .../src/test/resources/query/sql/query03.sql|  20 +
 .../src/test/resources/query/sql/query04.sql|  21 +
 .../src/test/resources/query/sql/query05.sql|  22 +
 .../src/test/resources/query/sql/query06.sql|  23 +
 .../src/test/resources/query/sql/query07.sql|  22 +
 .../src/test/resources/query/sql/query08.sql|  22 +
 .../src/test/resources/query/sql/query09.sql|  27 +
 .../src/test/resources/query/sql/query10.sql|  30 ++
 .../src/test/resources/query/sql/query11.sql|  30 ++
 .../src/test/resources/query/sql/query12.sql|  29 ++
 .../src/test/resources/query/sql/query13.sql|  28 +
 .../src/test/resources/query/sql/query14.sql|  36 ++
 .../src/test/resources/query/sql/query15.sql|  32 ++
 .../src/test/resources/query/sql/query16.sql|  35 ++
 .../src/test/resources/query/sql/query17.sql|  38 ++
 .../src/test/resources/query/sql/query18.sql|  34 ++
 .../src/test/resources/query/sql/query19.sql|  38 ++
 .../src/test/resources/query/sql/query20.sql|  34 ++
 .../src/test/resources/query/sql/query21.sql|  40 ++
 .../src/test/resources/query/sql/query22.sql|  42 ++
 .../src/test/resources/query/sql/query23.sql|  41 ++
 .../src/test/resources/query/sql/query24.sql|  45 ++
 .../src/test/resources/query/sql/query25.sql|  46 ++
 .../src/test/resources/query/sql/query26.sql|  48 ++
 .../src/test/resources/query/sql/query27.sql|  47 ++
 .../src/test/resources/query/sql/query28.sql|  30 ++
 .../src/test/resources/query/sql/query29.sql|  32 ++
 .../src/test/resources/query/sql/query30.sql|  19 +
 .../src/test/resources/query/sql/query31.sql|  40 ++
 .../src/test/resources/query/sql/query32.sql|  39 ++
 .../src/test/resources/query/sql/query33.sql|  43 ++
 .../src/test/resources/query/sql/query34.sql|  41 ++
 .../src/test/resources/query/sql/query35.sql|  41 ++

[03/13] kylin git commit: KYLIN-1356 move all it to a seprate module

2016-01-21 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/query/src/test/resources/query/sql_casewhen/query56.sql
--
diff --git a/query/src/test/resources/query/sql_casewhen/query56.sql 
b/query/src/test/resources/query/sql_casewhen/query56.sql
deleted file mode 100644
index 7006fbf..000
--- a/query/src/test/resources/query/sql_casewhen/query56.sql
+++ /dev/null
@@ -1,22 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
- SELECT (CASE WHEN ("TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" = 'Auction') THEN 
'Auction2' ELSE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" END) AS 
"LSTG_FORMAT_NAME__group_",
-  SUM("TEST_KYLIN_FACT"."PRICE") AS "sum_PRICE_ok"
-FROM "TEST_KYLIN_FACT" "TEST_KYLIN_FACT"
-GROUP BY (CASE WHEN ("TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" =  'Auction') THEN 
'Auction2' ELSE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" END)

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/query/src/test/resources/query/sql_derived/query01.sql
--
diff --git a/query/src/test/resources/query/sql_derived/query01.sql 
b/query/src/test/resources/query/sql_derived/query01.sql
deleted file mode 100644
index c4b0aae..000
--- a/query/src/test/resources/query/sql_derived/query01.sql
+++ /dev/null
@@ -1,47 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT 
- test_kylin_fact.seller_id 
- ,test_cal_dt.week_beg_dt 
- ,test_category_groupings.meta_categ_name 
- ,test_category_groupings.categ_lvl2_name 
- ,test_category_groupings.categ_lvl3_name 
- ,test_kylin_fact.lstg_format_name 
- ,test_sites.site_name 
- ,test_sites.site_id 
- ,test_sites.cre_user 
- ,sum(test_kylin_fact.price) as GMV, count(*) as TRANS_CNT 
- FROM test_kylin_fact 
- inner JOIN edw.test_cal_dt as test_cal_dt 
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
- inner JOIN test_category_groupings 
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id 
- inner JOIN edw.test_sites as test_sites 
- ON test_kylin_fact.lstg_site_id = test_sites.site_id 
- where test_kylin_fact.seller_id = 1002 
- group by 
- test_kylin_fact.seller_id 
- ,test_cal_dt.week_beg_dt 
- ,test_category_groupings.meta_categ_name 
- ,test_category_groupings.categ_lvl2_name 
- ,test_category_groupings.categ_lvl3_name 
- ,test_kylin_fact.lstg_format_name 
- ,test_sites.site_name 
- ,test_sites.site_id 
- ,test_sites.cre_user 

http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/query/src/test/resources/query/sql_derived/query02.sql
--
diff --git a/query/src/test/resources/query/sql_derived/query02.sql 
b/query/src/test/resources/query/sql_derived/query02.sql
deleted file mode 100644
index 027cd9c..000
--- a/query/src/test/resources/query/sql_derived/query02.sql
+++ /dev/null
@@ -1,47 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
---

kylin git commit: fix classnotdef

2016-01-21 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1356 cc50f9cd5 -> af5cd2b00


fix classnotdef


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af5cd2b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af5cd2b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af5cd2b0

Branch: refs/heads/KYLIN-1356
Commit: af5cd2b000d2e98d233a051e4e0300755ad8b65e
Parents: cc50f9c
Author: honma 
Authored: Fri Jan 22 11:40:08 2016 +0800
Committer: honma 
Committed: Fri Jan 22 11:40:08 2016 +0800

--
 kylin-it/pom.xml | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af5cd2b0/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 82e2d3e..7dccbc6 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -133,6 +133,11 @@
 ${project.parent.version}
 
 
+org.apache.maven
+maven-model
+test
+
+
 xerces
 xercesImpl
 test

[2/2] kylin git commit: minor: wrap exceptions in Observer with IOException

2016-01-21 Thread mahongbin

minor: wrap exceptions in Observer with IOException


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db87d462
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db87d462
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db87d462

Branch: refs/heads/2.0-rc
Commit: db87d46293ebe4c47284be02b09315a76e65edce
Parents: a118a27
Author: honma 
Authored: Fri Jan 22 11:17:21 2016 +0800
Committer: honma 
Committed: Fri Jan 22 11:19:33 2016 +0800

--
 .../observer/ObserverAggregationCache.java  | 23 
 1 file changed, 14 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db87d462/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index b7b12d1..d03fe16 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -31,8 +31,8 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
 import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
+import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
 
 /**
  * @author yangli9
@@ -67,15 +67,20 @@ public class ObserverAggregationCache extends 
AggregationCache {
 
 @Override
 public boolean next(List results) throws IOException {
-// AggregateRegionObserver.LOG.info("Kylin Scanner next()");
-boolean hasMore = false;
-if (iterator.hasNext()) {
-Entry entry = iterator.next();
-makeCells(entry, results);
-hasMore = iterator.hasNext();
+try {
+// AggregateRegionObserver.LOG.info("Kylin Scanner next()");
+boolean hasMore = false;
+if (iterator.hasNext()) {
+Entry entry = 
iterator.next();
+makeCells(entry, results);
+hasMore = iterator.hasNext();
+}
+// AggregateRegionObserver.LOG.info("Kylin Scanner next() 
done");
+
+return hasMore;
+} catch (Exception e) {
+throw new IOException("Error when calling next", e);
 }
-// AggregateRegionObserver.LOG.info("Kylin Scanner next() done");
-return hasMore;
 }
 
 private void makeCells(Entry entry, 
List results) {

[1/2] kylin git commit: Revert "KYLIN-1356 use exec-maven-plugin for IT environment provision"

2016-01-21 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc a118a27c6 -> 827a995da


Revert "KYLIN-1356 use exec-maven-plugin for IT environment provision"

This reverts commit ab853f9dc987549ac27dd2bb3db1732e3e474281.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/827a995d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/827a995d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/827a995d

Branch: refs/heads/2.0-rc
Commit: 827a995dab8ed28907d1e16abc5972ae253d712a
Parents: db87d46
Author: honma 
Authored: Fri Jan 22 11:18:21 2016 +0800
Committer: honma 
Committed: Fri Jan 22 11:19:33 2016 +0800

--
 .../engine/spark/BuildCubeWithSparkTest.java|   3 -
 .../apache/kylin/job/BuildCubeWithEngine.java   | 333 ---
 .../kylin/job/BuildCubeWithEngineTest.java  | 333 +++
 .../apache/kylin/job/BuildCubeWithStream.java   | 110 --
 .../kylin/job/BuildCubeWithStreamTest.java  | 114 +++
 .../apache/kylin/job/BuildIIWithEngineTest.java |   7 +-
 .../org/apache/kylin/job/BuildIIWithStream.java | 292 
 .../apache/kylin/job/BuildIIWithStreamTest.java | 290 
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   3 +
 pom.xml |  73 ++--
 10 files changed, 769 insertions(+), 789 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/827a995d/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
 
b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index f87fa99..f5b9741 100644
--- 
a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -46,13 +46,10 @@ import 
org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 
-//TODO: convert it to a normal class rather than a test case, like in 
BuildCubeWithEngine 
-@Ignore
 public class BuildCubeWithSparkTest {
 
 private CubeManager cubeManager;

http://git-wip-us.apache.org/repos/asf/kylin/blob/827a995d/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngine.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngine.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngine.java
deleted file mode 100644
index 2562a8b..000
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngine.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.engine.EngineFactory;
-import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import

kylin git commit: add fastBuildMode

2016-01-21 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-1356 c50d31448 -> 5f5bc29d5


add fastBuildMode


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f5bc29d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f5bc29d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f5bc29d

Branch: refs/heads/KYLIN-1356
Commit: 5f5bc29d5e6d8bf5db9e9e24838eb8952d672315
Parents: c50d314
Author: honma 
Authored: Fri Jan 22 15:16:25 2016 +0800
Committer: honma 
Committed: Fri Jan 22 15:16:25 2016 +0800

--
 kylin-it/pom.xml | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f5bc29d/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index fca8f6c..7112f21 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -31,9 +31,9 @@
 Kylin:IT
 
 
-
 
 
+
 
 
 
@@ -295,6 +295,7 @@
 java
 
 
-Dhdp.version=${hdp.version}
+
-DfastBuildMode=${fastBuildMode}
 -classpath
 
 
org.apache.kylin.provision.BuildCubeWithEngine
@@ -314,6 +315,7 @@
 java
 
 
-Dhdp.version=${hdp.version}
+
-DfastBuildMode=${fastBuildMode}
 -classpath
 
 
org.apache.kylin.provision.BuildCubeWithStream
@@ -333,6 +335,7 @@
 java
 
 
-Dhdp.version=${hdp.version}
+
-DfastBuildMode=${fastBuildMode}
 -classpath
 
 
org.apache.kylin.provision.BuildIIWithStream

kylin git commit: KYLIN-1297 correct LoadingCache settings

2016-01-20 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 2f5d89a8e -> eea691651


KYLIN-1297 correct LoadingCache settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eea69165
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eea69165
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eea69165

Branch: refs/heads/1.x-staging
Commit: eea6916518a59ce137ac6d26bd539e39bb871a6c
Parents: 2f5d89a
Author: honma 
Authored: Thu Jan 21 10:28:54 2016 +0800
Committer: honma 
Committed: Thu Jan 21 10:29:05 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  5 
 .../apache/kylin/dict/DictionaryManager.java| 31 +++-
 .../storage/hbase/CubeSegmentTupleIterator.java |  1 -
 3 files changed, 22 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index e0f774a..790e7a5 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -466,6 +466,11 @@ public class KylinConfig {
 return this.getOptional("kylin.server.mode", "all");
 }
 
+public int getCachedDictMaxEntrySize() {
+return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
+}
+
+
 public int getDictionaryMaxCardinality() {
 return 
Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "500"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index aa0bc5d..9294e03 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -28,9 +28,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -49,6 +46,10 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
 public class DictionaryManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryManager.class);
@@ -84,17 +85,19 @@ public class DictionaryManager {
 
 private DictionaryManager(KylinConfig config) {
 this.config = config;
-this.dictCache = 
CacheBuilder.newBuilder().weakValues().expireAfterWrite(10, 
TimeUnit.MINUTES).build(new CacheLoader() {
-@Override
-public DictionaryInfo load(String key) throws Exception {
-DictionaryInfo dictInfo = DictionaryManager.this.load(key, 
true);
-if (dictInfo == null) {
-return NONE_INDICATOR;
-} else {
-return dictInfo;
-}
-}
-});
+this.dictCache = 
CacheBuilder.newBuilder().maximumSize(KylinConfig.getInstanceFromEnv().getCachedDictMaxEntrySize())//
+.expireAfterWrite(1, TimeUnit.DAYS).build(new 
CacheLoader() {
+
+@Override
+public DictionaryInfo load(String key) throws Exception {
+DictionaryInfo dictInfo = 
DictionaryManager.this.load(key, true);
+if (dictInfo == null) {
+return NONE_INDICATOR;
+} else {
+return dictInfo;
+}
+}
+});
 }
 
 public Dictionary getDictionary(String resourcePath) throws IOException 
{

http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
--
diff --git

kylin git commit: fix issues in 2.0 upgrade, part 2

2016-01-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 4646cdb25 -> ae3bab684


fix issues in 2.0 upgrade, part 2


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae3bab68
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae3bab68
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae3bab68

Branch: refs/heads/2.0-rc
Commit: ae3bab68489a37652513029daac0c8a156b02f87
Parents: 4646cdb
Author: honma 
Authored: Tue Jan 19 13:48:35 2016 +0800
Committer: honma 
Committed: Tue Jan 19 13:48:41 2016 +0800

--
 .../java/org/apache/kylin/cube/cuboid/CuboidTest.java |  2 +-
 .../storage/hbase/cube/v2/CubeHBaseEndpointRPC.java   | 14 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ae3bab68/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index c17ee01..90ec655 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -148,7 +148,7 @@ public class CuboidTest extends LocalFileMetadataTestCase {
 Cuboid cuboid;
 
 cuboid = Cuboid.findById(cube, 0);
-assertEquals(toLong("100111000"), cuboid.getId());
+assertEquals(toLong("10100"), cuboid.getId());
 
 cuboid = Cuboid.findById(cube, 1);
 assertEquals(toLong("10111"), cuboid.getId());

http://git-wip-us.apache.org/repos/asf/kylin/blob/ae3bab68/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 5cca195..d6ef16c 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -255,11 +255,11 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 final ExpectedSizeIterator epResultItr = new 
ExpectedSizeIterator(scanRequests.size() * shardNum);
 
 for (final Pair epRange : 
getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
-for (int i = 0; i < scanRequests.size(); ++i) {
-final int scanIndex = i;
-executorService.submit(new Runnable() {
-@Override
-public void run() {
+executorService.submit(new Runnable() {
+@Override
+public void run() {
+for (int i = 0; i < scanRequests.size(); ++i) {
+int scanIndex = i;
 CubeVisitProtos.CubeVisitRequest.Builder builder = 
CubeVisitProtos.CubeVisitRequest.newBuilder();
 
builder.setGtScanRequest(scanRequestByteStrings.get(scanIndex)).setHbaseRawScan(rawScanByteStrings.get(scanIndex));
 for (IntList intList : hbaseColumnsToGTIntList) {
@@ -285,8 +285,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 }
 }
 }
-});
-}
+}
+});
 }
 
 return new EndpointResultsAsGTScanner(fullGTInfo, epResultItr, 
scanRequests.get(0).getColumns(), totalScannedCount.get());

[1/3] kylin git commit: enable filter in ResourceTool

2016-01-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 573c84e61 -> 000d883b1


enable filter in ResourceTool


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a7923b96
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a7923b96
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a7923b96

Branch: refs/heads/2.x-staging
Commit: a7923b9698c1ca0e9677759744ef8a64996b1aaa
Parents: 573c84e
Author: honma 
Authored: Tue Jan 19 15:05:38 2016 +0800
Committer: honma 
Committed: Tue Jan 19 15:05:38 2016 +0800

--
 .../kylin/common/persistence/ResourceTool.java  | 31 ++--
 1 file changed, 22 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a7923b96/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index c3038dd..4a23ba3 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -26,6 +26,7 @@ import org.apache.kylin.common.util.StringUtil;
 
 public class ResourceTool {
 
+private static String[] includes = null;
 private static String[] excludes = null;
 
 public static void main(String[] args) throws IOException {
@@ -38,6 +39,10 @@ public class ResourceTool {
 return;
 }
 
+String include = System.getProperty("include");
+if (include != null) {
+includes = include.split("\\s*,\\s*");
+}
 String exclude = System.getProperty("exclude");
 if (exclude != null) {
 excludes = exclude.split("\\s*,\\s*");
@@ -83,7 +88,7 @@ public class ResourceTool {
 
 // case of resource (not a folder)
 if (children == null) {
-if (matchExclude(path) == false) {
+if (matchFilter(path)) {
 RawResource res = src.getResource(path);
 if (res != null) {
 dst.putResource(path, res.inputStream, res.timestamp);
@@ -100,14 +105,22 @@ public class ResourceTool {
 }
 }
 
-private static boolean matchExclude(String path) {
-if (excludes == null)
-return false;
-for (String exclude : excludes) {
-if (path.startsWith(exclude))
-return true;
+private static boolean matchFilter(String path) {
+if (includes != null) {
+boolean in = false;
+for (String include : includes) {
+in = in || path.startsWith(include);
+}
+if (!in)
+return false;
+}
+if (excludes != null) {
+for (String exclude : excludes) {
+if (path.startsWith(exclude))
+return false;
+}
 }
-return false;
+return true;
 }
 
 public static void reset(KylinConfig config) throws IOException {
@@ -118,7 +131,7 @@ public class ResourceTool {
 private static void resetR(ResourceStore store, String path) throws 
IOException {
 ArrayList children = store.listResources(path);
 if (children == null) { // path is a resource (not a folder)
-if (matchExclude(path) == false) {
+if (matchFilter(path)) {
 store.deleteResource(path);
 }
 } else {

kylin git commit: bug fix in ResourceTool

2016-01-17 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc a09df13d3 -> fdba83ed4


bug fix in ResourceTool


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdba83ed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdba83ed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdba83ed

Branch: refs/heads/2.0-rc
Commit: fdba83ed4db98f4d927e75e13a1748bed928128c
Parents: a09df13
Author: honma 
Authored: Mon Jan 18 12:15:36 2016 +0800
Committer: honma 
Committed: Mon Jan 18 12:15:55 2016 +0800

--
 .../java/org/apache/kylin/common/persistence/ResourceTool.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fdba83ed/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index f13b723..4a23ba3 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -131,7 +131,7 @@ public class ResourceTool {
 private static void resetR(ResourceStore store, String path) throws 
IOException {
 ArrayList children = store.listResources(path);
 if (children == null) { // path is a resource (not a folder)
-if (matchFilter(path) == false) {
+if (matchFilter(path)) {
 store.deleteResource(path);
 }
 } else {

[2/3] kylin git commit: KYLIN-1297 avoid unnecessary converter creation

2016-01-14 Thread mahongbin

KYLIN-1297 avoid unnecessary converter creation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4ea1b39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4ea1b39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4ea1b39

Branch: refs/heads/2.x-staging
Commit: b4ea1b3915c70daf44a871a39d65282e9089b77c
Parents: 32cb630
Author: honma 
Authored: Thu Jan 14 18:34:35 2016 +0800
Committer: honma 
Committed: Thu Jan 14 18:34:35 2016 +0800

--
 .../storage/hbase/cube/v2/SequentialCubeTupleIterator.java | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4ea1b39/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
index 7e70a4d..dcc3aba 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
@@ -56,6 +56,7 @@ public class SequentialCubeTupleIterator implements 
ITupleIterator {
 
 @Override
 public boolean hasNext() {
+logger.info("hasNext called");
 if (next != null)
 return true;
 
@@ -75,7 +76,10 @@ public class SequentialCubeTupleIterator implements 
ITupleIterator {
 if (scannerIterator.hasNext()) {
 curScanner = scannerIterator.next();
 curRecordIterator = curScanner.iterator();
-curTupleConverter = new CubeTupleConverter(curScanner.cubeSeg, 
cuboid, selectedDimensions, selectedMetrics, tupleInfo);
+if (curRecordIterator.hasNext()) {
+//if the segment does not has any tuples, don't bother to 
create a converter
+curTupleConverter = new 
CubeTupleConverter(curScanner.cubeSeg, cuboid, selectedDimensions, 
selectedMetrics, tupleInfo);
+}
 } else {
 return false;
 }

[1/3] kylin git commit: don't do storage cleaning at buildcubewithengine

2016-01-14 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 08f42c0d5 -> d2bc9d140


don't do storage cleaning at buildcubewithengine


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32cb630d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32cb630d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32cb630d

Branch: refs/heads/2.x-staging
Commit: 32cb630d746e5f0948591fec0d5f52bbc8ef6dfb
Parents: 08f42c0
Author: honma 
Authored: Thu Jan 14 17:39:23 2016 +0800
Committer: honma 
Committed: Thu Jan 14 17:41:10 2016 +0800

--
 .../src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/32cb630d/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 46bf1bb..331e21d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -107,7 +107,6 @@ public class BuildCubeWithEngineTest {
 public void before() throws Exception {
 
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
 
-cleanupOldStorage();
 DeployUtil.initCliWorkDir();
 DeployUtil.deployMetadata();
 DeployUtil.overrideJobJarLocations();

[6/9] kylin git commit: KYLIN-1307 revisit growing dictionary

2016-01-13 Thread mahongbin

KYLIN-1307 revisit growing dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c5784b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c5784b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c5784b2

Branch: refs/heads/2.x-staging
Commit: 1c5784b28ff9bbeb4a1ee9d4db3b05461641165d
Parents: 142eaf4
Author: honma 
Authored: Tue Jan 12 16:06:39 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:52:57 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 .../apache/kylin/dict/DictionaryManager.java| 58 +++-
 .../apache/kylin/rest/service/CacheService.java |  2 +
 3 files changed, 51 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c5784b2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f6775e8..bfad306 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -412,7 +412,6 @@ public class KylinConfigBase implements Serializable {
 return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
 }
 
-
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }
@@ -453,6 +452,10 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
 
+public boolean isGrowingDictEnabled() {
+return 
Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
+}
+
 public int getHBaseScanMaxResultSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c5784b2/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 630b2b2..d49e43d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -140,26 +140,58 @@ public class DictionaryManager {
 
 initDictInfo(newDict, newDictInfo);
 
-DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
-if (largestDictInfo != null) {
-largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
-Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
-if (largestDictObject.contains(newDict)) {
-logger.info("dictionary content " + newDict + ", is contained 
by  dictionary at " + largestDictInfo.getResourcePath());
-return largestDictInfo;
-} else if (newDict.contains(largestDictObject)) {
-logger.info("dictionary content " + newDict + " is by far the 
largest, save it");
-return saveNewDict(newDictInfo);
+if (KylinConfig.getInstanceFromEnv().isGrowingDictEnabled()) {
+DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+if (largestDictInfo != null) {
+largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
+Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
+if (largestDictObject.contains(newDict)) {
+logger.info("dictionary content " + newDict + ", is 
contained by  dictionary at " + largestDictInfo.getResourcePath());
+return largestDictInfo;
+} else if (newDict.contains(largestDictObject)) {
+logger.info("dictionary content " + newDict + " is by far 
the largest, save it");
+return saveNewDict(newDictInfo);
+} else {
+logger.info("merge dict and save...");
+return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+}
 } else {
-logger.info("merge dict and save...");
-return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+logger.info("first dict of this column, save it

[9/9] kylin git commit: KYLIN-1308 maintain CubeHBaseScanRPC

2016-01-13 Thread mahongbin

KYLIN-1308 maintain CubeHBaseScanRPC


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc8ea781
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc8ea781
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc8ea781

Branch: refs/heads/2.x-staging
Commit: bc8ea78171856e6c438b1a5da5ed9e2b1a49a862
Parents: fe19e3d
Author: honma 
Authored: Wed Jan 13 17:42:54 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:54:04 2016 +0800

--
 .../apache/kylin/cube/kv/LazyRowKeyEncoder.java |  26 ++
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |  43 +++
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 348 +++
 .../hbase/cube/v2/CubeSegmentScanner.java   |   2 +-
 4 files changed, 266 insertions(+), 153 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bc8ea781/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java
index 0fca726..c93f65e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/LazyRowKeyEncoder.java
@@ -18,9 +18,15 @@
 
 package org.apache.kylin.cube.kv;
 
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ShardingHash;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * A LazyRowKeyEncoder will not try to calculate shard
  * It works for both enableSharding or non-enableSharding scenario
@@ -38,4 +44,24 @@ public class LazyRowKeyEncoder extends RowKeyEncoder {
 throw new RuntimeException("If enableSharding false, you should 
never calculate shard");
 }
 }
+
+
+//for non-sharding cases it will only return one byte[] with not shard at 
beginning
+public List getRowKeysDifferentShards(byte[] halfCookedKey) {
+final short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());
+
+if (!enableSharding) {
+return Lists.newArrayList(halfCookedKey);//not shard to append at 
head, so it is already well cooked
+} else {
+List ret = Lists.newArrayList();
+for (short i = 0; i < cuboidShardNum; ++i) {
+short shard = 
ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), i, 
cubeSeg.getTotalShards());
+byte[] cookedKey = Arrays.copyOf(halfCookedKey, 
halfCookedKey.length);
+BytesUtil.writeShort(shard, cookedKey, 0, 
RowConstants.ROWKEY_SHARDID_LEN);
+ret.add(cookedKey);
+}
+return ret;
+}
+
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc8ea781/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 07143b1..db39455 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -30,6 +30,7 @@ import org.apache.kylin.gridtable.IGTScanner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
@@ -104,6 +105,48 @@ public abstract class CubeHBaseRPC {
 return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, 
hbaseCaching, hbaseMaxResultSize);
 }
 
+protected List preparedHBaseScans(GTRecord pkStart, GTRecord 
pkEnd, List fuzzyKeys, ImmutableBitSet selectedColBlocks) {
+final List> selectedColumns = 
makeHBaseColumns(selectedColBlocks);
+List ret = Lists.newArrayList();
+
+LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
+byte[] start = encoder.createBuf();
+byte[] end = encoder.createBuf();
+List startKeys;
+List endKeys;
+
+encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);
+encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start);
+startKeys = encoder.getRowKeysDifferentShards(start);
+
+encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);
+encoder.encode(pkEnd,

[1/9] kylin git commit: KYLIN-1297 correct LoadingCache settings

2016-01-13 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 05b7ca560 -> bc8ea7817


KYLIN-1297 correct LoadingCache settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2d3d8597
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2d3d8597
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2d3d8597

Branch: refs/heads/2.x-staging
Commit: 2d3d859724f0f8fde2113ef187e14635e52dfd00
Parents: 05b7ca5
Author: honma 
Authored: Fri Jan 8 15:48:58 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:48:31 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java  | 5 +
 .../src/main/java/org/apache/kylin/dict/DictionaryManager.java  | 5 +++--
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2d3d8597/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 96c76f0..f6775e8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -408,6 +408,11 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }
 
+public int getCachedDictMaxEntrySize() {
+return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
+}
+
+
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2d3d8597/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index cb58721..cca8341 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -83,7 +83,8 @@ public class DictionaryManager {
 
 private DictionaryManager(KylinConfig config) {
 this.config = config;
-this.dictCache = 
CacheBuilder.newBuilder().weakValues().expireAfterWrite(10, 
TimeUnit.MINUTES).build(new CacheLoader() {
+this.dictCache = 
CacheBuilder.newBuilder().maximumSize(KylinConfig.getInstanceFromEnv().getCachedDictMaxEntrySize())//
+.expireAfterWrite(1, TimeUnit.DAYS).build(new 
CacheLoader() {
 @Override
 public DictionaryInfo load(String key) throws Exception {
 DictionaryInfo dictInfo = DictionaryManager.this.load(key, 
true);
@@ -357,7 +358,7 @@ public class DictionaryManager {
 DictionaryInfo load(String resourcePath, boolean loadDictObj) throws 
IOException {
 ResourceStore store = MetadataManager.getInstance(config).getStore();
 
-//logger.debug("Going to load DictionaryInfo from " + resourcePath);
+logger.debug("Loading DictionaryInfo(loadDictObj:" + loadDictObj + ") 
from " + resourcePath);
 DictionaryInfo info = store.getResource(resourcePath, 
DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : 
DictionaryInfoSerializer.INFO_SERIALIZER);
 
 //if (loadDictObj)

[5/9] kylin git commit: KYLIN-1301 TRUE/FALSE filter pruning

2016-01-13 Thread mahongbin

KYLIN-1301 TRUE/FALSE filter pruning


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/142eaf41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/142eaf41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/142eaf41

Branch: refs/heads/2.x-staging
Commit: 142eaf41150a421d18f3fb584f58cff4df2d16da
Parents: a3a6a4d
Author: honma 
Authored: Wed Jan 13 17:50:41 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:51:17 2016 +0800

--
 .../kylin/gridtable/GTScanRangePlanner.java | 12 ++---
 .../kylin/gridtable/DictGridTableTest.java  | 21 +++
 query/src/test/resources/query/sql/query97.sql  | 28 
 3 files changed, 58 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/142eaf41/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 27fa6b4..6ff1ab4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -18,6 +18,7 @@ import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.common.FuzzyValueCombination;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -49,7 +50,6 @@ public class GTScanRangePlanner {
  * @param partitionColRef the TblColRef in GT
  */
 public GTScanRangePlanner(GTInfo info, Pair 
segmentStartAndEnd, TblColRef partitionColRef) {
-  
 
 this.info = info;
 this.segmentStartAndEnd = segmentStartAndEnd;
@@ -188,7 +188,9 @@ public class GTScanRangePlanner {
 throw new IllegalStateException("Filter should be AND instead 
of " + andFilter);
 
 Collection andRanges = 
translateToAndDimRanges(andFilter.getChildren());
-result.add(andRanges);
+if (andRanges != null) {
+result.add(andRanges);
+}
 }
 
 return preEvaluateConstantConditions(result);
@@ -198,7 +200,11 @@ public class GTScanRangePlanner {
 Map rangeMap = new HashMap();
 for (TupleFilter filter : andFilters) {
 if ((filter instanceof CompareTupleFilter) == false) {
-continue;
+if (filter instanceof ConstantTupleFilter && 
!filter.evaluate(null, null)) {
+return null;
+} else {
+continue;
+}
 }
 
 CompareTupleFilter comp = (CompareTupleFilter) filter;

http://git-wip-us.apache.org/repos/asf/kylin/blob/142eaf41/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 4588051..df69c17 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -89,6 +89,7 @@ public class DictGridTableTest {
 ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, 
"20"));
 ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, 
"30"));
 ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 
1, "30"));
+
 }
 
 @Test
@@ -138,6 +139,26 @@ public class DictGridTableTest {
 assertEquals("[142119360, null]-[null, null]", 
r.get(0).toString());
 assertEquals(0, r.get(0).fuzzyKeys.size());
 }
+{
+//skip FALSE filter
+LogicalTupleFilter filter = and(ageComp1, 
ConstantTupleFilter.FALSE);
+List r = planner.planScanRanges(filter);
+assertEquals(0, r.size());
+}
+{
+//TRUE or FALSE filter
+LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, 
ConstantTupleFilter.FALSE);
+List r = planner.planScanRanges(filter);
+assertEquals(1, r.size());
+assertEquals("[null, null]-[null, null]", r.get(0).toString());

kylin git commit: add reviewboardrc

2016-01-13 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 65566bf13 -> 5d2a58a1d


add reviewboardrc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d2a58a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d2a58a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d2a58a1

Branch: refs/heads/2.x-staging
Commit: 5d2a58a1d8dc8458c57072e97680aef0fe752cc9
Parents: 65566bf
Author: honma 
Authored: Thu Jan 14 10:25:05 2016 +0800
Committer: honma 
Committed: Thu Jan 14 10:38:00 2016 +0800

--
 .reviewboardrc | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d2a58a1/.reviewboardrc
--
diff --git a/.reviewboardrc b/.reviewboardrc
new file mode 100644
index 000..7fcc51f
--- /dev/null
+++ b/.reviewboardrc
@@ -0,0 +1,4 @@
+REVIEWBOARD_URL = "https://reviews.apache.org;
+REPOSITORY = "kylin"
+BRANCH = "2.x-staging"
+LAND_DEST_BRANCH = "2.x-staging"

kylin git commit: add reviewboardrc

2016-01-13 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 721729823 -> bd73b643d


add reviewboardrc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd73b643
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd73b643
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd73b643

Branch: refs/heads/1.x-staging
Commit: bd73b643d307dcacf3e2d99e71f3b8f7008f1d1b
Parents: 7217298
Author: honma 
Authored: Thu Jan 14 10:59:03 2016 +0800
Committer: honma 
Committed: Thu Jan 14 10:59:07 2016 +0800

--
 .reviewboardrc | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bd73b643/.reviewboardrc
--
diff --git a/.reviewboardrc b/.reviewboardrc
new file mode 100644
index 000..79d75f5
--- /dev/null
+++ b/.reviewboardrc
@@ -0,0 +1,4 @@
+REVIEWBOARD_URL = "https://reviews.apache.org;
+REPOSITORY = "kylin"
+BRANCH = "1.x-staging"
+LAND_DEST_BRANCH = "1.x-staging"

kylin git commit: KYLIN-1221 Kill underlying running hadoop job while discard a job

2016-01-13 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 5d2a58a1d -> cef83f01c


KYLIN-1221 Kill underlying running hadoop job while discard a job

Bugs closed: KYLIN-1221

Reviewed at https://reviews.apache.org/r/42285/


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cef83f01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cef83f01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cef83f01

Branch: refs/heads/2.x-staging
Commit: cef83f01c5f538cdacadbd5148f28c901148aa7d
Parents: 5d2a58a
Author: Dong Li 
Authored: Thu Jan 14 10:48:05 2016 +0800
Committer: honma 
Committed: Thu Jan 14 10:48:05 2016 +0800

--
 .../engine/mr/common/MapReduceExecutable.java   | 29 
 1 file changed, 18 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cef83f01/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index 585e0e1..f9c4673 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -53,12 +53,11 @@ import com.google.common.base.Preconditions;
  */
 public class MapReduceExecutable extends AbstractExecutable {
 
+public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
 private static final String KEY_MR_JOB = "MR_JOB_CLASS";
 private static final String KEY_PARAMS = "MR_JOB_PARAMS";
 private static final String KEY_COUNTER_SAVEAS = "MR_COUNTER_SAVEAS";
 
-public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
-
 public MapReduceExecutable() {
 super();
 }
@@ -117,7 +116,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 try {
 //for async mr job, ToolRunner just return 0;
 ToolRunner.run(hadoopJob, args);
-
+
 if (hadoopJob.isSkipped()) {
 return new ExecuteResult(ExecuteResult.State.SUCCEED, 
"skipped");
 }
@@ -169,7 +168,15 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 }
 
Thread.sleep(context.getConfig().getYarnStatusCheckIntervalSeconds() * 1000);
 }
-//TODO kill discarded mr job using "hadoop job -kill " + mrJobId
+
+// try to kill running map-reduce job to release resources.
+if (job != null) {
+try {
+job.killJob();
+} catch (Exception e) {
+logger.warn("failed to kill hadoop job: " + 
job.getJobID(), e);
+}
+}
 
 return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
 
@@ -211,7 +218,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 logger.info("kylin.job.yarn.app.rest.check.status.url" + " is not 
set, read from job configuration");
 }
 String rmWebHost = 
HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, 
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, job.getConfiguration());
-if(HAUtil.isHAEnabled(job.getConfiguration())) {
+if (HAUtil.isHAEnabled(job.getConfiguration())) {
 YarnConfiguration conf = new 
YarnConfiguration(job.getConfiguration());
 String active = RMHAUtils.findActiveRMHAId(conf);
 rmWebHost = 
HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS,
 active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
@@ -236,22 +243,22 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 addExtraInfo(MAP_REDUCE_WAIT_TIME, t + "");
 }
 
-public void setMapReduceJobClass(Class 
clazzName) {
-setParam(KEY_MR_JOB, clazzName.getName());
-}
-
 public String getMapReduceJobClass() throws ExecuteException {
 return getParam(KEY_MR_JOB);
 }
 
-public void setMapReduceParams(String param) {
-setParam(KEY_PARAMS, param);
+public void setMapReduceJobClass(Class 
clazzName) {
+setParam(KEY_MR_JOB, clazzName.getName());
 }
 
 public String getMapReduceParams() {
 return getParam(KEY_PARAMS);
 }
 
+public void setMapReduceParams(String param) {
+setParam(KEY_PARAMS, param);
+}
+
 public String getCounterSaveAs() {
 return getParam(KEY_COUNTER_SAVEAS);
 }

kylin git commit: add reviewboardrc

2016-01-13 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 5feca4e7c -> b5a444534


add reviewboardrc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5a44453
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5a44453
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5a44453

Branch: refs/heads/2.0-rc
Commit: b5a4445346bc576aa1b29e276e8a46112f3d54ce
Parents: 5feca4e
Author: honma 
Authored: Thu Jan 14 10:56:19 2016 +0800
Committer: honma 
Committed: Thu Jan 14 10:56:24 2016 +0800

--
 .reviewboardrc | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b5a44453/.reviewboardrc
--
diff --git a/.reviewboardrc b/.reviewboardrc
new file mode 100644
index 000..b6c24fa
--- /dev/null
+++ b/.reviewboardrc
@@ -0,0 +1,4 @@
+REVIEWBOARD_URL = "https://reviews.apache.org;
+REPOSITORY = "kylin"
+BRANCH = "2.0-rc"
+LAND_DEST_BRANCH = "2.0-rc"

kylin git commit: KYLIN-1307 revisit growing dictionary

2016-01-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 3904a1767 -> 9f743fb78


KYLIN-1307 revisit growing dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9f743fb7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9f743fb7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9f743fb7

Branch: refs/heads/2.0-rc
Commit: 9f743fb7871d02f2c06e8fe21d288de1f502c939
Parents: 3904a17
Author: honma 
Authored: Tue Jan 12 16:06:39 2016 +0800
Committer: honma 
Committed: Tue Jan 12 16:06:39 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 .../apache/kylin/dict/DictionaryManager.java| 58 +++-
 .../apache/kylin/rest/service/CacheService.java |  2 +
 3 files changed, 51 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a5ca690..4c640f3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -412,7 +412,6 @@ public class KylinConfigBase implements Serializable {
 return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
 }
 
-
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }
@@ -453,6 +452,10 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
 
+public boolean isGrowingDictEnabled() {
+return 
Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
+}
+
 public int getHBaseScanMaxResultSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index b3cd634..cd12f01 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -139,26 +139,58 @@ public class DictionaryManager {
 
 initDictInfo(newDict, newDictInfo);
 
-DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
-if (largestDictInfo != null) {
-largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
-Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
-if (largestDictObject.contains(newDict)) {
-logger.info("dictionary content " + newDict + ", is contained 
by  dictionary at " + largestDictInfo.getResourcePath());
-return largestDictInfo;
-} else if (newDict.contains(largestDictObject)) {
-logger.info("dictionary content " + newDict + " is by far the 
largest, save it");
-return saveNewDict(newDictInfo);
+if (KylinConfig.getInstanceFromEnv().isGrowingDictEnabled()) {
+DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+if (largestDictInfo != null) {
+largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
+Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
+if (largestDictObject.contains(newDict)) {
+logger.info("dictionary content " + newDict + ", is 
contained by  dictionary at " + largestDictInfo.getResourcePath());
+return largestDictInfo;
+} else if (newDict.contains(largestDictObject)) {
+logger.info("dictionary content " + newDict + " is by far 
the largest, save it");
+return saveNewDict(newDictInfo);
+} else {
+logger.info("merge dict and save...");
+return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+}
 } else {
-logger.info("merge dict and save...");
-return mergeDictionary(Lists.newArrayList(newDictInfo,

kylin git commit: KYLIN-1301 fix IT

2016-01-11 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 28b2154fd -> 3904a1767


KYLIN-1301 fix IT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3904a176
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3904a176
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3904a176

Branch: refs/heads/2.0-rc
Commit: 3904a17671ba150728f9dbc40d1504d76401f9e6
Parents: 28b2154
Author: honma 
Authored: Mon Jan 11 16:57:52 2016 +0800
Committer: honma 
Committed: Mon Jan 11 16:57:52 2016 +0800

--
 query/src/test/resources/query/sql/query96.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3904a176/query/src/test/resources/query/sql/query96.sql
--
diff --git a/query/src/test/resources/query/sql/query96.sql 
b/query/src/test/resources/query/sql/query96.sql
index a1eda6f..c9c0c6f 100644
--- a/query/src/test/resources/query/sql/query96.sql
+++ b/query/src/test/resources/query/sql/query96.sql
@@ -24,5 +24,5 @@ inner JOIN edw.test_cal_dt as test_cal_dt
  ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND 
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
  inner JOIN edw.test_sites as test_sites
  ON test_kylin_fact.lstg_site_id = test_sites.site_id
- where test_kylin_fact.lstg_format_name='XXXUnknownFP-GTC' 
+ where test_kylin_fact.lstg_format_name='XXXUnknown' 
  group by test_cal_dt.cal_dt

kylin git commit: KYLIN-1301 fix segment pruning failure in 2.x versions

2016-01-10 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc f91e738a7 -> edab5a433


KYLIN-1301 fix segment pruning failure in 2.x versions


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/edab5a43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/edab5a43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/edab5a43

Branch: refs/heads/2.0-rc
Commit: edab5a433cb3260065ebddb6869505175db30758
Parents: f91e738
Author: honma 
Authored: Mon Jan 11 11:25:44 2016 +0800
Committer: honma 
Committed: Mon Jan 11 11:25:44 2016 +0800

--
 .../kylin/gridtable/GTScanRangePlanner.java   | 13 +++--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java   |  4 
 .../kylin/storage/hbase/cube/v2/CubeHBaseRPC.java | 18 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java   |  5 -
 .../storage/hbase/cube/v2/CubeSegmentScanner.java | 14 ++
 5 files changed, 38 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/edab5a43/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index f0e42a0..dad0f7c 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -43,7 +43,16 @@ public class GTScanRangePlanner {
 final private RecordComparator rangeEndComparator;
 final private RecordComparator rangeStartEndComparator;
 
+/**
+ * @param info
+ * @param segmentStartAndEnd in GT encoding
+ * @param partitionColRef the TblColRef in GT
+ */
 public GTScanRangePlanner(GTInfo info, Pair 
segmentStartAndEnd, TblColRef partitionColRef) {
+if (partitionColRef != null && segmentStartAndEnd == null) {
+throw new IllegalArgumentException("segmentStartAndEnd not 
provided when partitionColRef is set to " + partitionColRef);
+}
+
 this.info = info;
 this.segmentStartAndEnd = segmentStartAndEnd;
 this.partitionColRef = partitionColRef;
@@ -91,9 +100,9 @@ public class GTScanRangePlanner {
 List fuzzyKeys;
 
 for (ColumnRange range : andDimRanges) {
-
 if (partitionColRef != null && 
range.column.equals(partitionColRef)) {
-
+logger.debug("Pre-check partition col filter, partitionColRef 
{}, segmentstartandend {}, range begin {}, range end {}",//
+new Object[] { partitionColRef, segmentStartAndEnd, 
range.begin, range.end });
 if 
(rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(), 
range.end) <= 0 //
 && 
rangeStartEndComparator.comparator.compare(range.begin, 
segmentStartAndEnd.getSecond()) < 0) {
 //segment range is [Closed,Open)

http://git-wip-us.apache.org/repos/asf/kylin/blob/edab5a43/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 6ff07a4..a193d10 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -63,9 +63,13 @@ import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.HBaseZeroCopyByteString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
+public static final Logger logger = 
LoggerFactory.getLogger(CubeHBaseEndpointRPC.class);
+
 private static ExecutorService executorService = 
Executors.newCachedThreadPool();
 
 static class EndpointResultsAsGTScanner implements IGTScanner {

http://git-wip-us.apache.org/repos/asf/kylin/blob/edab5a43/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 1335af0..d319c2d 100644
---

kylin git commit: KYLIN-1294 disabled related test cases as it is depending on Calcite-980

2016-01-07 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging a1b26ff9f -> bb130af0a


KYLIN-1294 disabled related test cases as it is depending on Calcite-980


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bb130af0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bb130af0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bb130af0

Branch: refs/heads/2.x-staging
Commit: bb130af0a37f36fbc76eb9a549b222df964e7761
Parents: a1b26ff
Author: honma 
Authored: Fri Jan 8 10:50:41 2016 +0800
Committer: honma 
Committed: Fri Jan 8 10:50:41 2016 +0800

--
 .../metadata/filter/ConstantTupleFilter.java|  9 ++
 .../metadata/filter/LogicalTupleFilter.java | 19 +++--
 query/src/test/resources/query/sql/query92.sql  | 30 
 .../resources/query/sql/query92.sql.disabled| 30 
 query/src/test/resources/query/sql/query93.sql  | 30 
 .../resources/query/sql/query93.sql.disabled| 30 
 query/src/test/resources/query/sql/query94.sql  | 30 
 .../resources/query/sql/query94.sql.disabled| 30 
 query/src/test/resources/query/sql/query95.sql  | 30 
 .../resources/query/sql/query95.sql.disabled| 30 
 10 files changed, 146 insertions(+), 122 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bb130af0/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index cc3add2..3056a9c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -54,6 +54,15 @@ public class ConstantTupleFilter extends TupleFilter {
 }
 
 @Override
+public TupleFilter reverse() {
+if (this.evaluate(null, null)) {
+return ConstantTupleFilter.FALSE;
+} else {
+return ConstantTupleFilter.TRUE;
+}
+}
+
+@Override
 public void addChild(TupleFilter child) {
 throw new UnsupportedOperationException("This is " + this + " and 
child is " + child);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb130af0/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 83404eb..0929cf1 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -46,12 +46,27 @@ public class LogicalTupleFilter extends TupleFilter {
 return cloneTuple;
 }
 
+
+//private TupleFilter reverseNestedNots(TupleFilter filter, int depth) 
{
+//if ((filter instanceof LogicalTupleFilter) && (filter.operator 
== FilterOperatorEnum.NOT)) {
+//assert (filter.children.size() == 1);
+//return reverseNestedNots(filter.children.get(0), depth + 1);
+//}
+//
+//if (depth % 2 == 1) {
+//return filter;
+//} else {
+//return filter.reverse();
+//}
+//}
+
+
 @Override
 public TupleFilter reverse() {
 switch (operator) {
 case NOT:
-assert (children.size() == 1);
-return children.get(0);
+throw new IllegalStateException("not( not in ()) is invalid 
syntax");
+//return reverseNestedNots(this, 0);
 case AND:
 case OR:
 LogicalTupleFilter reverse = new 
LogicalTupleFilter(REVERSE_OP_MAP.get(operator));

http://git-wip-us.apache.org/repos/asf/kylin/blob/bb130af0/query/src/test/resources/query/sql/query92.sql
--
diff --git a/query/src/test/resources/query/sql/query92.sql 
b/query/src/test/resources/query/sql/query92.sql
deleted file mode 100644
index e551a45..000
--- a/query/src/test/resources/query/sql/query92.sql
+++ /dev/null
@@ -1,30 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional

[2/2] kylin git commit: KYLIN-1293 keep kylin log clean

2016-01-07 Thread mahongbin

KYLIN-1293 keep kylin log clean


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50b3f429
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50b3f429
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50b3f429

Branch: refs/heads/2.x-staging
Commit: 50b3f429d557a748198824a79483c77252a0a483
Parents: c1e83b0
Author: honma 
Authored: Fri Jan 8 11:01:55 2016 +0800
Committer: honma 
Committed: Fri Jan 8 11:03:06 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|   3 +-
 .../apache/kylin/common/KylinConfigBase.java|  28 +-
 .../kylin/common/persistence/ResourceStore.java |   2 +-
 .../apache/kylin/common/util/BytesSplitter.java |  42 -
 .../kylin/common/util/CompressionUtils.java |   4 +-
 .../common/util/MemoryBudgetController.java |  10 +-
 .../src/main/resources/kylin-log4j.properties   |   2 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   2 +-
 .../inmemcubing/AbstractInMemCubeBuilder.java   |   2 +-
 .../cube/inmemcubing/InMemCubeBuilder.java  |  10 +-
 .../org/apache/kylin/cube/util/CubingUtils.java |   1 -
 .../apache/kylin/dict/DictionaryManager.java|  16 +-
 .../apache/kylin/query/routing/QueryRouter.java |   5 +-
 .../apache/kylin/query/routing/RoutingRule.java |   7 +-
 .../kylin/rest/controller/UserController.java   |   6 +-
 .../kylin/rest/service/BadQueryDetector.java|   9 +-
 .../apache/kylin/rest/service/BasicService.java |  24 +-
 .../apache/kylin/rest/service/CacheService.java |  14 +-
 .../apache/kylin/rest/service/QueryService.java |   2 +-
 .../resources/kylin-server-log4j.properties |  97 +--
 .../kylin/storage/hbase/HBaseResourceStore.java | 762 +--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   8 +-
 22 files changed, 508 insertions(+), 548 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/50b3f429/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index bc18989..81f5827 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -99,8 +99,7 @@ public class KylinConfig extends KylinConfigBase {
 throw new IllegalStateException("Metadata uri : " + 
metaUri + " is not a valid REST URI address");
 }
 } catch (Exception e) {
-logger.info(e.getLocalizedMessage());
-throw new IllegalStateException("Metadata uri : " + metaUri + " is 
not recognized");
+throw new IllegalStateException("Metadata uri : " + metaUri + " is 
not recognized", e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/50b3f429/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e7c373e..96c76f0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -44,7 +44,7 @@ import com.google.common.collect.Sets;
 public class KylinConfigBase implements Serializable {
 
 private static final Logger logger = 
LoggerFactory.getLogger(KylinConfigBase.class);
-
+
 /*
  * DON'T DEFINE CONSTANTS FOR PROPERTY KEYS!
  * 
@@ -62,9 +62,9 @@ public class KylinConfigBase implements Serializable {
 }
 
 // 

-
+
 private volatile Properties properties = new Properties();
-
+
 public String getOptional(String prop) {
 return getOptional(prop, null);
 }
@@ -90,7 +90,7 @@ public class KylinConfigBase implements Serializable {
 }
 return r;
 }
-
+
 /**
  * Use with care, properties should be read-only. This is for testing 
mostly.
  */
@@ -98,7 +98,7 @@ public class KylinConfigBase implements Serializable {
 logger.info("Kylin Config was updated with " + key + " : " + value);
 properties.setProperty(key, value);
 }
-
+
 protected Properties getAllProperties() {
 return properties;
 }
@@ -114,9 +114,9 @@ public class KylinConfigBase implements Serializable {
 }
 this.properties = newProperties;
 }
-
+
 // 

-
+
 public String

[1/2] kylin git commit: KYLIN-1270 bug fix by removing formatTs's affections

2016-01-05 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging f0a8078a0 -> f7057e87c


KYLIN-1270 bug fix by removing formatTs's affections


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/280fce6c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/280fce6c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/280fce6c

Branch: refs/heads/2.x-staging
Commit: 280fce6cc817f85b4ebdbe71513fb3b4b7d329c3
Parents: f0a8078
Author: honma 
Authored: Tue Jan 5 10:43:56 2016 +0800
Committer: honma 
Committed: Tue Jan 5 16:10:58 2016 +0800

--
 .../kylin/source/kafka/TimedJsonStreamParser.java   | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/280fce6c/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index c1d8379..0907623 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -65,7 +65,7 @@ public final class TimedJsonStreamParser extends 
StreamingParser {
 private static final Logger logger = 
LoggerFactory.getLogger(TimedJsonStreamParser.class);
 
 private List allColumns;
-private boolean formatTs = false;
+private boolean formatTs = false;//not used
 private final ObjectMapper mapper = new ObjectMapper();
 private String tsColName = "timestamp";
 private final JavaType mapType = MapType.construct(HashMap.class, 
SimpleType.construct(String.class), SimpleType.construct(String.class));
@@ -119,26 +119,26 @@ public final class TimedJsonStreamParser extends 
StreamingParser {
 String columnName = column.getName();
 if (columnName.equalsIgnoreCase("minute_start")) {
 normalized = TimeUtil.getMinuteStart(t);
-result.add(formatTs ? 
DateFormat.formatToTimeStr(normalized) : String.valueOf(normalized));
+result.add(DateFormat.formatToTimeStr(normalized));
 } else if (columnName.equalsIgnoreCase("hour_start")) {
 normalized = TimeUtil.getHourStart(t);
-result.add(formatTs ? 
DateFormat.formatToTimeStr(normalized) : String.valueOf(normalized));
+result.add(DateFormat.formatToTimeStr(normalized));
 } else if (columnName.equalsIgnoreCase("day_start")) {
 //from day_start on, formatTs will output date format
 normalized = TimeUtil.getDayStart(t);
-result.add(formatTs ? 
DateFormat.formatToDateStr(normalized) : String.valueOf(normalized));
+result.add(DateFormat.formatToDateStr(normalized));
 } else if (columnName.equalsIgnoreCase("week_start")) {
 normalized = TimeUtil.getWeekStart(t);
-result.add(formatTs ? 
DateFormat.formatToDateStr(normalized) : String.valueOf(normalized));
+result.add(DateFormat.formatToDateStr(normalized));
 } else if (columnName.equalsIgnoreCase("month_start")) {
 normalized = TimeUtil.getMonthStart(t);
-result.add(formatTs ? 
DateFormat.formatToDateStr(normalized) : String.valueOf(normalized));
+result.add(DateFormat.formatToDateStr(normalized));
 } else if (columnName.equalsIgnoreCase("quarter_start")) {
 normalized = TimeUtil.getQuarterStart(t);
-result.add(formatTs ? 
DateFormat.formatToDateStr(normalized) : String.valueOf(normalized));
+result.add(DateFormat.formatToDateStr(normalized));
 } else if (columnName.equalsIgnoreCase("year_start")) {
 normalized = TimeUtil.getYearStart(t);
-result.add(formatTs ? 
DateFormat.formatToDateStr(normalized) : String.valueOf(normalized));
+result.add(DateFormat.formatToDateStr(normalized));
 } else {
 String x = root.get(columnName.toLowerCase());
 result.add(x);

[1/2] kylin git commit: bug fix: remove checkDupByInfo in merge dict

2016-01-05 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc ea702c8a0 -> 5516466b3


bug fix: remove checkDupByInfo in merge dict


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5516466b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5516466b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5516466b

Branch: refs/heads/2.0-rc
Commit: 5516466b30d0db8a38a64d622877e3b81bfa55f2
Parents: 1dc361c
Author: honma 
Authored: Tue Jan 5 16:11:29 2016 +0800
Committer: honma 
Committed: Wed Jan 6 10:05:29 2016 +0800

--
 .../java/org/apache/kylin/dict/DictionaryManager.java | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5516466b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 981e19a..78b4307 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -199,11 +199,11 @@ public class DictionaryManager {
 signature.setLastModifiedTime(System.currentTimeMillis());
 signature.setPath("merged_with_no_original_path");
 
-String dupDict = checkDupByInfo(newDictInfo);
-if (dupDict != null) {
-logger.info("Identical dictionary input " + newDictInfo.getInput() 
+ ", reuse existing dictionary at " + dupDict);
-return getDictionaryInfo(dupDict);
-}
+//String dupDict = checkDupByInfo(newDictInfo);
+//if (dupDict != null) {
+//logger.info("Identical dictionary input " + 
newDictInfo.getInput() + ", reuse existing dictionary at " + dupDict);
+//return getDictionaryInfo(dupDict);
+//}
 
 //check for cases where merging dicts are actually same
 boolean identicalSourceDicts = true;

[2/2] kylin git commit: Revert "KYLIN-1270"

2016-01-05 Thread mahongbin

Revert "KYLIN-1270"

This reverts commit d3648849626c78b1e44df4050a620c25e5ee805f.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1dc361c6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1dc361c6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1dc361c6

Branch: refs/heads/2.0-rc
Commit: 1dc361c6d30d1b01e0bf2c3677f6a6d90f670396
Parents: ea702c8
Author: honma 
Authored: Tue Jan 5 16:10:45 2016 +0800
Committer: honma 
Committed: Wed Jan 6 10:05:29 2016 +0800

--
 .../org/apache/kylin/source/kafka/TimedJsonStreamParser.java| 5 -
 1 file changed, 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1dc361c6/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index f7f15ab..0907623 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -36,7 +36,6 @@ package org.apache.kylin.source.kafka;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -109,10 +108,8 @@ public final class TimedJsonStreamParser extends 
StreamingParser {
 //" cannot be null, the message offset is " + 
messageAndOffset.getOffset() + " content is " + new 
String(messageAndOffset.getRawData()));
 long t;
 if (StringUtils.isEmpty(tsStr)) {
-logger.info("tsStr not found");
 t = 0;
 } else {
-logger.info("tsStr is :" + tsStr);
 t = Long.valueOf(tsStr);
 }
 ArrayList result = Lists.newArrayList();
@@ -148,8 +145,6 @@ public final class TimedJsonStreamParser extends 
StreamingParser {
 }
 }
 
-logger.info(Arrays.toString(result.toArray()));
-
 return new StreamingMessage(result, messageAndOffset.offset(), t, 
Collections. emptyMap());
 
 } catch (IOException e) {

[1/3] kylin git commit: KYLIN-1283 Replace GTScanRequest's SerDer form Kryo to manual

2016-01-04 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 4d4e743c7 -> 2ac2f56a1


http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac2f56a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 6981214..6acaa8b 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.common.util.CompressionUtils;
 import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.util.KryoUtils;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.IGTScanner;
@@ -132,8 +131,8 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 try {
 this.serviceStartTime = System.currentTimeMillis();
 
-GTScanRequest scanReq = 
KryoUtils.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getGtScanRequest()),
 GTScanRequest.class);
-RawScan hbaseRawScan = 
KryoUtils.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getHbaseRawScan()),
 RawScan.class);
+GTScanRequest scanReq = 
GTScanRequest.serializer.deserialize(ByteBuffer.wrap(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getGtScanRequest(;
+RawScan hbaseRawScan = 
RawScan.serializer.deserialize(ByteBuffer.wrap(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getHbaseRawScan(;
 List hbaseColumnsToGT = Lists.newArrayList();
 for (IntList intList : request.getHbaseColumnsToGTList()) {
 hbaseColumnsToGT.add(intList.getIntsList());

http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac2f56a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
index 10e80ae..76c6637 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
@@ -34,7 +34,6 @@ import org.apache.kylin.measure.hllc.HLLCMeasureType;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.datatype.LongMutable;
 import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
@@ -214,7 +213,7 @@ public class EndpointAggregators {
 }
 return length;
 }
-
+
 public List deserializeMetricValues(ByteBuffer buffer) {
 List ret = Lists.newArrayList();
 for (int i = 0; i < measureSerializers.length; i++) {
@@ -226,7 +225,7 @@ public class EndpointAggregators {
 }
 
 public static byte[] serialize(EndpointAggregators o) {
-ByteBuffer buf = 
ByteBuffer.allocate(CoprocessorConstants.SERIALIZE_BUFFER_SIZE);
+ByteBuffer buf = 
ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
 serializer.serialize(o, buf);
 byte[] result = new byte[buf.position()];
 System.arraycopy(buf.array(), 0, result, 0, buf.position());
@@ -237,9 +236,7 @@ public class EndpointAggregators {
 return serializer.deserialize(ByteBuffer.wrap(bytes));
 }
 
-private static final Serializer serializer = new Serializer();
-
-private static class Serializer implements 
BytesSerializer {
+private static final BytesSerializer serializer = new 
BytesSerializer() {
 
 @Override
 public void serialize(EndpointAggregators value, ByteBuffer out) {
@@ -278,7 +275,7 @@ public class EndpointAggregators {
 return new EndpointAggregators(funcNames, dataTypes, infos, 
tableInfo);
 }
 
-}
+};
 
 public int getMeasureSerializeLength() {
 int length = 0;

http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac2f56a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java

[2/3] kylin git commit: KYLIN-1283 Replace GTScanRequest's SerDer form Kryo to manual

2016-01-04 Thread mahongbin

KYLIN-1283 Replace GTScanRequest's SerDer form Kryo to manual


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ac2f56a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ac2f56a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ac2f56a

Branch: refs/heads/2.x-staging
Commit: 2ac2f56a138e9c014796423bcddb1a01248fe218
Parents: ec5c28d
Author: honma 
Authored: Wed Dec 30 22:20:24 2015 +0800
Committer: honma 
Committed: Mon Jan 4 21:39:54 2016 +0800

--
 .../org/apache/kylin/common/util/ByteArray.java |  10 ++
 .../kylin/common/util/ImmutableBitSet.java  |  34 --
 .../org/apache/kylin/common/util/BasicTest.java |  10 --
 .../kylin/cube/gridtable/CubeCodeSystem.java|  96 ++-
 .../kylin/cube/gridtable/FixLenSerializer.java  | 103 
 .../cube/gridtable/TrimmedCubeCodeSystem.java   | 103 ++--
 .../cube/inmemcubing/InMemCubeBuilder.java  |   7 +-
 .../java/org/apache/kylin/gridtable/GTInfo.java | 100 
 .../org/apache/kylin/gridtable/GTRecord.java|   9 +-
 .../kylin/gridtable/GTSampleCodeSystem.java |  12 ++
 .../org/apache/kylin/gridtable/GTScanRange.java |   2 +
 .../kylin/gridtable/GTScanRangePlanner.java |   7 +-
 .../apache/kylin/gridtable/GTScanRequest.java   |  84 --
 .../java/org/apache/kylin/gridtable/GTUtil.java |  10 ++
 .../inmemcubing/ConcurrentDiskStoreTest.java|   2 +-
 .../cube/inmemcubing/MemDiskStoreTest.java  |   2 +-
 .../gridtable/AggregationCacheSpillTest.java|   6 +-
 .../kylin/gridtable/DictGridTableTest.java  |  41 +--
 .../kylin/gridtable/GTScanReqSerDerTest.java| 116 +++
 .../kylin/gridtable/SimpleGridTableTest.java|   4 +-
 .../metadata/datatype/BigDecimalSerializer.java |   6 +-
 .../kylin/metadata/datatype/DataType.java   |  30 -
 .../index/TableRecordInfoDigest.java|   8 +-
 .../coprocessor/CoprocessorConstants.java   |  26 -
 .../coprocessor/CoprocessorProjector.java   |   9 +-
 .../common/coprocessor/CoprocessorRowType.java  |  10 +-
 .../observer/ObserverAggregators.java   |  13 +--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  18 ++-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |   8 +-
 .../hbase/cube/v2/CubeSegmentScanner.java   |   7 +-
 .../storage/hbase/cube/v2/CubeStorageQuery.java |   1 -
 .../kylin/storage/hbase/cube/v2/RawScan.java|  56 -
 .../coprocessor/endpoint/CubeVisitService.java  |   5 +-
 .../endpoint/EndpointAggregators.java   |  11 +-
 .../ii/coprocessor/endpoint/IIEndpoint.java |   7 +-
 35 files changed, 702 insertions(+), 271 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac2f56a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index ccd5001..015d5c5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -105,6 +105,16 @@ public class ByteArray implements Comparable, 
Serializable {
 return copy;
 }
 
+//notice this will have a length header
+public void exportData(ByteBuffer out) {
+BytesUtil.writeByteArray(this.data, this.offset, this.length, out);
+}
+
+public static ByteArray importData(ByteBuffer in) {
+byte[] bytes = BytesUtil.readByteArray(in);
+return new ByteArray(bytes);
+}
+
 public void copyFrom(ByteArray other) {
 System.arraycopy(other.array(), other.offset, data, offset, 
other.length);
 this.length = other.length;

http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac2f56a/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
index f5a22d2..a36bcef 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
@@ -17,6 +17,7 @@
  */
 package org.apache.kylin.common.util;
 
+import java.nio.ByteBuffer;
 import java.util.BitSet;
 
 public class ImmutableBitSet {
@@ -30,6 +31,16 @@ public class ImmutableBitSet {
 this(newBitSet(index));
 }
 
+public ImmutableBitSet(BitSet set) {
+this.set = (BitSet) set.clone();
+this.arr = new

[3/3] kylin git commit: KYLIN-1270 improve TimedJsonStreamParser to support week_start, month_start, quarter_start, year_start

2016-01-04 Thread mahongbin

KYLIN-1270 improve TimedJsonStreamParser to support week_start, month_start, 
quarter_start, year_start


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ec5c28d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ec5c28d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ec5c28d9

Branch: refs/heads/2.x-staging
Commit: ec5c28d99d58c761b7dd6a0be2301fc8563f156c
Parents: 4d4e743
Author: honma 
Authored: Wed Dec 30 21:33:37 2015 +0800
Committer: honma 
Committed: Mon Jan 4 21:39:54 2016 +0800

--
 .../org/apache/kylin/common/util/TimeUtil.java  | 44 +++--
 .../apache/kylin/common/util/TimeUtilTest.java  | 21 +++-
 .../source/kafka/TimedJsonStreamParser.java | 51 ++--
 3 files changed, 94 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ec5c28d9/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
index c79e88b..17868a6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -1,12 +1,13 @@
 package org.apache.kylin.common.util;
 
+import java.util.Calendar;
+import java.util.TimeZone;
+
 /**
  */
 public class TimeUtil {
-public enum NormalizedTimeUnit {
-MINUTE, HOUR, DAY
-}
 
+private static TimeZone gmt = TimeZone.getTimeZone("GMT");
 private static long ONE_MINUTE_TS = 60 * 1000;
 private static long ONE_HOUR_TS = 60 * ONE_MINUTE_TS;
 private static long ONE_DAY_TS = 24 * ONE_HOUR_TS;
@@ -23,7 +24,40 @@ public class TimeUtil {
 return ts / ONE_DAY_TS * ONE_DAY_TS;
 }
 
-public static long getNextPeriodStart(long ts, long period) {
-return ((ts + period - 1) / period) * period;
+public static long getWeekStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(getDayStart(ts));
+calendar.add(Calendar.DAY_OF_WEEK, calendar.getFirstDayOfWeek() - 
calendar.get(Calendar.DAY_OF_WEEK));
+return calendar.getTimeInMillis();
 }
+
+public static long getMonthStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+int month = calendar.get(Calendar.MONTH);
+calendar.clear();
+calendar.set(year, month, 1);
+return calendar.getTimeInMillis();
+}
+
+public static long getQuarterStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+int month = calendar.get(Calendar.MONTH);
+calendar.clear();
+calendar.set(year, month / 3 * 3, 1);
+return calendar.getTimeInMillis();
+}
+
+public static long getYearStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+calendar.clear();
+calendar.set(year, 0, 1);
+return calendar.getTimeInMillis();
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ec5c28d9/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
index d81d49a..3fdf6aa 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
@@ -3,14 +3,19 @@ package org.apache.kylin.common.util;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.TimeZone;
 
-import org.apache.kylin.common.util.TimeUtil.NormalizedTimeUnit;
 import org.junit.Assert;
 import org.junit.Test;
 
 /**
  */
 public class TimeUtilTest {
+
+public enum NormalizedTimeUnit {
+MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR
+}
+
 public static long normalizeTime(long timeMillis, NormalizedTimeUnit unit) 
{
 Calendar a = Calendar.getInstance();
 Calendar b = Calendar.getInstance();
@@ -28,6 +33,7 @@ public class TimeUtilTest {
 @Test
 public void basicTest() throws ParseException {
 java.text.DateFormat dateFormat = new SimpleDateFormat("/MM/dd 
HH:mm:ss");
+

kylin git commit: KYLIN-1270 improve TimedJsonStreamParser to support week_start, month_start, quarter_start, year_start

2015-12-30 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc e0748f5b4 -> 5c83a14a4


KYLIN-1270 improve TimedJsonStreamParser to support week_start, month_start, 
quarter_start, year_start


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c83a14a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c83a14a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c83a14a

Branch: refs/heads/2.0-rc
Commit: 5c83a14a4bea5e1496d49ed362593aaaf84922ed
Parents: e0748f5
Author: honma 
Authored: Wed Dec 30 21:33:37 2015 +0800
Committer: honma 
Committed: Wed Dec 30 21:35:43 2015 +0800

--
 .../org/apache/kylin/common/util/TimeUtil.java  | 44 +++--
 .../apache/kylin/common/util/TimeUtilTest.java  | 21 +++-
 .../source/kafka/TimedJsonStreamParser.java | 51 ++--
 3 files changed, 94 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5c83a14a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
index c79e88b..17868a6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -1,12 +1,13 @@
 package org.apache.kylin.common.util;
 
+import java.util.Calendar;
+import java.util.TimeZone;
+
 /**
  */
 public class TimeUtil {
-public enum NormalizedTimeUnit {
-MINUTE, HOUR, DAY
-}
 
+private static TimeZone gmt = TimeZone.getTimeZone("GMT");
 private static long ONE_MINUTE_TS = 60 * 1000;
 private static long ONE_HOUR_TS = 60 * ONE_MINUTE_TS;
 private static long ONE_DAY_TS = 24 * ONE_HOUR_TS;
@@ -23,7 +24,40 @@ public class TimeUtil {
 return ts / ONE_DAY_TS * ONE_DAY_TS;
 }
 
-public static long getNextPeriodStart(long ts, long period) {
-return ((ts + period - 1) / period) * period;
+public static long getWeekStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(getDayStart(ts));
+calendar.add(Calendar.DAY_OF_WEEK, calendar.getFirstDayOfWeek() - 
calendar.get(Calendar.DAY_OF_WEEK));
+return calendar.getTimeInMillis();
 }
+
+public static long getMonthStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+int month = calendar.get(Calendar.MONTH);
+calendar.clear();
+calendar.set(year, month, 1);
+return calendar.getTimeInMillis();
+}
+
+public static long getQuarterStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+int month = calendar.get(Calendar.MONTH);
+calendar.clear();
+calendar.set(year, month / 3 * 3, 1);
+return calendar.getTimeInMillis();
+}
+
+public static long getYearStart(long ts) {
+Calendar calendar = Calendar.getInstance(gmt);
+calendar.setTimeInMillis(ts);
+int year = calendar.get(Calendar.YEAR);
+calendar.clear();
+calendar.set(year, 0, 1);
+return calendar.getTimeInMillis();
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5c83a14a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
index d81d49a..3fdf6aa 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
@@ -3,14 +3,19 @@ package org.apache.kylin.common.util;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
+import java.util.TimeZone;
 
-import org.apache.kylin.common.util.TimeUtil.NormalizedTimeUnit;
 import org.junit.Assert;
 import org.junit.Test;
 
 /**
  */
 public class TimeUtilTest {
+
+public enum NormalizedTimeUnit {
+MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR
+}
+
 public static long normalizeTime(long timeMillis, NormalizedTimeUnit unit) 
{
 Calendar a = Calendar.getInstance();
 Calendar b = Calendar.getInstance();
@@ -28,6 +33,7 @@ public class TimeUtilTest {
 @Test
 public void basicTest() throws ParseException {
 java.text.DateFormat dateFormat

kylin git commit: KYLIN-1239 attribute each htable with team contact and owner name

2015-12-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 0de917283 -> 6ec2adbbe


KYLIN-1239 attribute each htable with team contact and owner name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ec2adbb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ec2adbb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ec2adbb

Branch: refs/heads/2.0-rc
Commit: 6ec2adbbe3f26fb78c18b26f61a9c7f587ed34c0
Parents: 0de9172
Author: honma 
Authored: Thu Dec 17 11:31:55 2015 +0800
Committer: honma 
Committed: Tue Dec 22 16:10:36 2015 +0800

--
 build/conf/kylin.properties  |  5 +
 .../org/apache/kylin/common/KylinConfig.java |  6 ++
 .../realization/IRealizationConstants.java   |  5 -
 .../apache/kylin/engine/spark/SparkCubing.java   |  3 +--
 .../test_case_data/localmeta/kylin.properties|  5 +
 examples/test_case_data/sandbox/kylin.properties |  5 +
 .../storage/hbase/steps/CreateHTableJob.java |  4 +---
 .../storage/hbase/steps/CubeHTableUtil.java  | 19 ---
 .../hbase/steps/HBaseStreamingOutput.java|  2 +-
 .../kylin/storage/hbase/util/CleanHtableCLI.java |  6 +-
 10 files changed, 49 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 2c34e92..f2170bd 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -17,6 +17,11 @@
 
 ## Config for Kylin Engine ##
 
+
+# optional information for the owner of kylin platform, it can be your team's 
email
+# currently it will be attached to each kylin's htable attribute
+kylin.owner=who...@kylin.apache.org
+
 # List of web servers in use, this enables one web server instance to sync up 
with other servers.
 kylin.rest.servers=localhost:7070
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index eb76534..c65ade4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -55,6 +55,8 @@ public class KylinConfig implements Serializable {
  * NOTE: These key constants should be private or even better be removed.
  *   All external access should go through public methods.
  */
+public static final String KYLIN_OWNER = "kylin.owner";
+
 public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
 
 public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
@@ -706,6 +708,10 @@ public class KylinConfig implements Serializable {
 return this.getOptional(HIVE_DATABASE_FOR_INTERMEDIATE_TABLE, 
"default");
 }
 
+public String getKylinOwner() {
+return this.getOptional(KYLIN_OWNER, "");
+}
+
 public void setRunAsRemoteCommand(String v) {
 properties.setProperty(KYLIN_JOB_RUN_AS_REMOTE_CMD, v);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 7cbcdf8..138a209 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -31,7 +31,10 @@ public class IRealizationConstants {
  * which kylin server(represented by its kylin_metadata prefix) owns this 
htable
  */
 public final static String HTableTag = "KYLIN_HOST";
-
+
+public final static String HTableOwner = "OWNER";
+
+public final static String HTableUser = "USER";
 
 public final static String HTableCreationTime = "CREATION_TIME";
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 01d97fd..9a2610c 100644
---

kylin git commit: fix typo

2015-12-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 3fc3883a4 -> 7849a47ff


fix typo


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7849a47f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7849a47f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7849a47f

Branch: refs/heads/2.x-staging
Commit: 7849a47ffbc8d7d5dac42a46b3579aa4a7275a09
Parents: 3fc3883
Author: honma 
Authored: Tue Dec 22 15:24:07 2015 +0800
Committer: honma 
Committed: Tue Dec 22 16:11:12 2015 +0800

--
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7849a47f/README.md
--
diff --git a/README.md b/README.md
index 73313f5..10bc863 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ Get Help
 The fastest way to get response from our developers is to send email to our 
mail list ,   
 and remember to subscribe our mail list via 

 
-Licnese
+License
 
 Please refer to 
[LICENSE](https://github.com/apache/incubator-kylin/blob/master/LICENSE) file.

[3/6] kylin git commit: KYLIN-976 Support Custom Aggregation Types

2015-12-15 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/c721d679/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
 
b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
deleted file mode 100644
index d0aff5a..000
--- 
a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-
-/**
- * @author yangli9
- * 
- */
-public class BigDecimalMinAggregator extends MeasureAggregator {
-
-BigDecimal max = null;
-
-@Override
-public void reset() {
-max = null;
-}
-
-@Override
-public void aggregate(BigDecimal value) {
-if (max == null)
-max = value;
-else if (max.compareTo(value) > 0)
-max = value;
-}
-
-@Override
-public BigDecimal getState() {
-return max;
-}
-
-@Override
-public int getMemBytes() {
-return guessBigDecimalMemBytes();
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/c721d679/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
 
b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
deleted file mode 100644
index 3c858b2..000
--- 
a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- * @author yangli9
- * 
- */
-public class BigDecimalSerializer extends MeasureSerializer {
-
-@Override
-public void serialize(BigDecimal value, ByteBuffer out) {
-byte[] bytes = value.unscaledValue().toByteArray();
-
-BytesUtil.writeVInt(value.scale(), out);
-BytesUtil.writeVInt(bytes.length, out);
-out.put(bytes);
-}
-
-@Override
-public BigDecimal deserialize(ByteBuffer in) {
-int scale = BytesUtil.readVInt(in);
-int n = BytesUtil.readVInt(in);
-
-byte[] bytes = new byte[n];
-in.get(bytes);
-
-return new BigDecimal(new BigInteger(bytes), scale);
-}
-
-@Override
-public BigDecimal valueOf(byte[] value) {
-if (value == null)
-return new BigDecimal(0);
-else
-return new BigDecimal(Bytes.toString(value));
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/c721d679/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
 
b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
deleted file mode 100644

[08/10] kylin git commit: KYLIN-919 fix ui show dimension error

2015-12-04 Thread mahongbin

KYLIN-919 fix ui show dimension error

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24d3825b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24d3825b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24d3825b

Branch: refs/heads/2.x-staging
Commit: 24d3825bc4652f2f449dde68223788aa3b51fefe
Parents: 5216fad
Author: jian 
Authored: Fri Dec 4 12:29:09 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/24d3825b/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 2920458..ae4a0f1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -47,6 +47,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return temp;
   };
 
+  //get columns from model
   $scope.getDimColumnsByTable = function (tableName) {
 if (!tableName) {
   return [];
@@ -55,6 +56,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 var tableDim = _.find($scope.metaModel.model.dimensions, function 
(dimension) {
   return dimension.table == tableName
 });
+if(!tableDim){
+  return [];
+}
 var tableDimColumns = tableDim.columns;
 var avaColObject = _.filter(tableColumns, function (col) {
   return tableDimColumns.indexOf(col.name) != -1;

[02/10] kylin git commit: KYLIN-1179 Cannot use String as partition column

2015-12-04 Thread mahongbin

KYLIN-1179 Cannot use String as partition column

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/225a7d5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/225a7d5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/225a7d5b

Branch: refs/heads/2.x-staging
Commit: 225a7d5b6c68e4c0d5a70283823e4fede59d5522
Parents: 9f60f47
Author: lidongsjtu 
Authored: Mon Nov 30 10:48:36 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:06 2015 +0800

--
 .../hbase/cube/v2/CubeSegmentScanner.java   | 26 +++-
 webapp/app/js/controllers/modelEdit.js  |  2 +-
 2 files changed, 16 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/225a7d5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 1eed318..2f78a86 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -11,6 +11,7 @@ import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.DateFormat;
@@ -30,6 +31,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.GTUtil;
 import org.apache.kylin.gridtable.IGTScanner;
 import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.DataType;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -83,27 +85,23 @@ public class CubeSegmentScanner implements IGTScanner {
 GTInfo trimmedInfo = GTInfo.deserialize(trimmedInfoBytes);
 
 for (GTScanRange range : scanRanges) {
-scanRequests.add(new GTScanRequest(trimmedInfo, 
range.replaceGTInfo(trimmedInfo),
-gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, 
gtFilter, allowPreAggregate));
+scanRequests.add(new GTScanRequest(trimmedInfo, 
range.replaceGTInfo(trimmedInfo), gtDimensions, gtAggrGroups, gtAggrMetrics, 
gtAggrFuncs, gtFilter, allowPreAggregate));
 }
 
 scanner = new Scanner();
 }
 
 private Pair getSegmentStartAndEnd(TblColRef 
tblColRef, int index) {
-
-String partitionColType = tblColRef.getColumnDesc().getDatatype();
-
 ByteArray start;
 if (cubeSeg.getDateRangeStart() != Long.MIN_VALUE) {
-start = translateTsToString(cubeSeg.getDateRangeStart(), 
partitionColType, index);
+start = translateTsToString(cubeSeg.getDateRangeStart(), index, 1);
 } else {
 start = new ByteArray();
 }
 
 ByteArray end;
 if (cubeSeg.getDateRangeEnd() != Long.MAX_VALUE) {
-end = translateTsToString(cubeSeg.getDateRangeEnd(), 
partitionColType, index);
+end = translateTsToString(cubeSeg.getDateRangeEnd(), index, -1);
 } else {
 end = new ByteArray();
 }
@@ -111,19 +109,25 @@ public class CubeSegmentScanner implements IGTScanner {
 
 }
 
-private ByteArray translateTsToString(long ts, String partitionColType, 
int index) {
+private ByteArray translateTsToString(long ts, int index, int 
roundingFlag) {
 String value;
-if ("date".equalsIgnoreCase(partitionColType)) {
+DataType partitionColType = info.getColumnType(index);
+if (partitionColType.isDate()) {
 value = DateFormat.formatToDateStr(ts);
-} else if ("timestamp".equalsIgnoreCase(partitionColType)) {
+} else if (partitionColType.isDatetime()) {
 //TODO: if partition col is not dict encoded, value's format may 
differ from expected. Though by default it is not the case
 value = DateFormat.formatToTimeWithoutMilliStr(ts);
+} else if (partitionColType.isStringFamily()) {
+String partitionDateFormat = 
cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat();
+if (StringUtils.isEmpty(partitionDateFormat))
+partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
+value = DateFormat.formatToDateStr(ts, partitionDateFormat);
 } else {
 throw new RuntimeException("Type "

[10/10] kylin git commit: KYLIN-919 model name validate when edit model, add lowercase check

2015-12-04 Thread mahongbin

KYLIN-919 model name validate when edit model,add lowercase check

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4afd144a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4afd144a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4afd144a

Branch: refs/heads/2.x-staging
Commit: 4afd144af38f99c2fc7333b18f07f2880cd80434
Parents: 24d3825
Author: Zhong 
Authored: Wed Dec 2 15:02:03 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 webapp/app/js/controllers/modelSchema.js | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4afd144a/webapp/app/js/controllers/modelSchema.js
--
diff --git a/webapp/app/js/controllers/modelSchema.js 
b/webapp/app/js/controllers/modelSchema.js
index 9e9f09c..682ce5e 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -166,9 +166,11 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, 
QueryService, UserServi
 
 var modelName = $scope.modelsManager.selectedModel.name.toUpperCase();
 var models = $scope.modelsManager.modelNameList;
-if ($scope.modelMode=="addNewModel"&(modelName) != -1 || 
models.indexOf(modelName.toLowerCase()) !=-1) {
-  SweetAlert.swal('', "Model named [" + modelName + "] already exist!", 
'warning');
-  return false;
+if ($scope.modelMode=="addNewModel") {
+  if(models.indexOf(modelName) != -1 || 
models.indexOf(modelName.toLowerCase()) !=-1){
+SweetAlert.swal('', "Model named [" + modelName + "] already exist!", 
'warning');
+return false;
+  }
 }
 return true;
   }

[07/10] kylin git commit: minor, update hyperlinks for kylin contacts on webapp

2015-12-04 Thread mahongbin

minor, update hyperlinks for kylin contacts on webapp

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d10c20dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d10c20dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d10c20dd

Branch: refs/heads/2.x-staging
Commit: d10c20ddea345e3acae21c8d0f2d372f419153ff
Parents: e8e2d27
Author: lidongsjtu 
Authored: Thu Dec 3 11:38:11 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 webapp/app/partials/footer.html | 4 ++--
 webapp/app/partials/login.html  | 2 +-
 webapp/app/partials/models/models_tree.html | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d10c20dd/webapp/app/partials/footer.html
--
diff --git a/webapp/app/partials/footer.html b/webapp/app/partials/footer.html
index 5a72a52..d0a513a 100644
--- a/webapp/app/partials/footer.html
+++ b/webapp/app/partials/footer.html
@@ -20,8 +20,8 @@
 
 
 
-http://kylin.io; style="color:#808080;"> Home Page |
-https://groups.google.com/forum/#!forum/kylin-olap; 
style="color:#808080;"> Google Group
+http://kylin.apache.org; style="color:#808080;"> Apache Kylin |
+http://kylin.apache.org/community/; 
style="color:#808080;"> Apache Kylin Community
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d10c20dd/webapp/app/partials/login.html
--
diff --git a/webapp/app/partials/login.html b/webapp/app/partials/login.html
index e64f9d5..79128b1 100644
--- a/webapp/app/partials/login.html
+++ b/webapp/app/partials/login.html
@@ -56,7 +56,7 @@
 
 
 
-https://groups.google.com/forum/#!forum/kylin-olap; target="_blank" 
class="text-muted">Login Issue?
+http://kylin.apache.org/community/; target="_blank" 
class="text-muted">Login Issue?
 
 
 {{loading? 
'Logging In...': 'Log In'}}

http://git-wip-us.apache.org/repos/asf/kylin/blob/d10c20dd/webapp/app/partials/models/models_tree.html
--
diff --git a/webapp/app/partials/models/models_tree.html 
b/webapp/app/partials/models/models_tree.html
index 30706cf..7b94841 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -26,7 +26,7 @@
 
 
   
- New 
Model
+New 
Model
   
   
 New Cube

[04/10] kylin git commit: KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0

2015-12-04 Thread mahongbin

KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5216fadd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5216fadd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5216fadd

Branch: refs/heads/2.x-staging
Commit: 5216faddd68df289be590f2f214b84a41011694f
Parents: d10c20d
Author: lidongsjtu 
Authored: Thu Dec 3 11:44:59 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 .../job/upgrade/CubeDescSignatureUpdate.java|  66 ---
 .../job/upgrade/v2/DataModelDescUpgradeV2.java  | 170 +++
 .../kylin/metadata/model/DataModelDesc.java |   8 +
 3 files changed, 217 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5216fadd/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
 
b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
index 178f1b0..36784de 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
@@ -18,57 +18,40 @@
 
 package org.apache.kylin.job.upgrade;
 
-import com.google.common.collect.Lists;
+import java.util.List;
+
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
 
-import java.util.List;
+import com.google.common.collect.Lists;
 
 /**
  * Created by dongli on 11/17/15.
  */
 public class CubeDescSignatureUpdate {
+private static final Log logger = 
LogFactory.getLog(CubeDescSignatureUpdate.class);
 private KylinConfig config = null;
 private ResourceStore store;
 private String[] cubeNames;
 private List updatedResources = Lists.newArrayList();
 private List errorMsgs = Lists.newArrayList();
 
-private static final Log logger = 
LogFactory.getLog(CubeDescSignatureUpdate.class);
-
 public CubeDescSignatureUpdate(String[] cubes) {
 config = KylinConfig.getInstanceFromEnv();
 store = ResourceStore.getStore(config);
 cubeNames = cubes;
 }
 
-public void update() {
-logger.info("Reloading Cube Metadata from store: " + 
store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
-CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
-List cubeDescs;
-if (ArrayUtils.isEmpty(cubeNames)) {
-cubeDescs = cubeDescManager.listAllDesc();
-} else {
-String[] names = cubeNames[0].split(",");
-if (ArrayUtils.isEmpty(names))
-return;
-cubeDescs = Lists.newArrayListWithCapacity(names.length);
-for (String name : names) {
-cubeDescs.add(cubeDescManager.getCubeDesc(name));
-}
-}
-for (CubeDesc cubeDesc : cubeDescs) {
-updateCubeDesc(cubeDesc);
-}
-}
-
 public static void main(String args[]) {
-if (args != null && args.length != 0 && args.length != 1) {
+if (args != null && args.length > 1) {
 System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; 
e.g, cube1,cube2 ");
 return;
 }
@@ -100,11 +83,40 @@ public class CubeDescSignatureUpdate {
 
logger.info("=");
 }
 
+public void update() {
+logger.info("Reloading Cube Metadata from store: " + 
store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+List cubeDescs;
+if (ArrayUtils.isEmpty(cubeNames)) {
+cubeDescs = cubeDescManager.listAllDesc();
+} else {
+String[] names = cubeNames[0].split(",");
+if (ArrayUtils.isEmpty(names))
+return;
+cubeDescs = Lists.newArrayListWithCapacity(names.length);
+for (String name : names) {
+cubeDescs.add(cubeDescManager.getCubeDesc(name));
+}
+}
+for (CubeDesc cubeDesc : cubeDescs) {
+

[05/10] kylin git commit: KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery

2015-12-04 Thread mahongbin

KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2fb097bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2fb097bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2fb097bb

Branch: refs/heads/2.0-rc
Commit: 2fb097bba79414b70b8a025b893a9ebf59a11258
Parents: fde4a3d
Author: lidongsjtu 
Authored: Thu Dec 3 09:49:38 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:14:34 2015 +0800

--
 .../apache/kylin/query/relnode/OLAPLimitRel.java |  7 +--
 .../resources/query/sql_verifyCount/query04.sql  | 19 +++
 .../query/sql_verifyCount/query04.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query05.sql  | 19 +++
 .../query/sql_verifyCount/query05.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query06.sql  | 19 +++
 .../query/sql_verifyCount/query06.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query07.sql  | 19 +++
 .../query/sql_verifyCount/query07.sql.disable| 19 ---
 9 files changed, 81 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 74d5de0..572a5c7 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -102,8 +102,11 @@ public class OLAPLimitRel extends SingleRel implements 
OLAPRel {
 
 @Override
 public EnumerableRel implementEnumerable(List inputs) {
-return new EnumerableLimit(getCluster(), 
getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
-sole(inputs), localOffset, localFetch);
+EnumerableRel input = sole(inputs);
+if (input instanceof OLAPRel) {
+((OLAPRel) input).replaceTraitSet(EnumerableConvention.INSTANCE);
+}
+return EnumerableLimit.create(input, localOffset, localFetch);
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query04.sql
--
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql 
b/query/src/test/resources/query/sql_verifyCount/query04.sql
new file mode 100644
index 000..9d3e409
--- /dev/null
+++ b/query/src/test/resources/query/sql_verifyCount/query04.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select * from test_kylin_fact limit 100

http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
--
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable 
b/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
deleted file mode 100644
index 9d3e409..000
--- a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,

[08/10] kylin git commit: KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0

2015-12-04 Thread mahongbin

KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ecbfe72
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ecbfe72
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ecbfe72

Branch: refs/heads/2.0-rc
Commit: 3ecbfe72d94547d9cd3d4c90c98024c8fc915ce5
Parents: 1b6f1fe
Author: lidongsjtu 
Authored: Thu Dec 3 11:44:59 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:14:52 2015 +0800

--
 .../job/upgrade/CubeDescSignatureUpdate.java|  66 ---
 .../job/upgrade/v2/DataModelDescUpgradeV2.java  | 170 +++
 .../kylin/metadata/model/DataModelDesc.java |   8 +
 3 files changed, 217 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ecbfe72/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
 
b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
index 178f1b0..36784de 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
@@ -18,57 +18,40 @@
 
 package org.apache.kylin.job.upgrade;
 
-import com.google.common.collect.Lists;
+import java.util.List;
+
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
 
-import java.util.List;
+import com.google.common.collect.Lists;
 
 /**
  * Created by dongli on 11/17/15.
  */
 public class CubeDescSignatureUpdate {
+private static final Log logger = 
LogFactory.getLog(CubeDescSignatureUpdate.class);
 private KylinConfig config = null;
 private ResourceStore store;
 private String[] cubeNames;
 private List updatedResources = Lists.newArrayList();
 private List errorMsgs = Lists.newArrayList();
 
-private static final Log logger = 
LogFactory.getLog(CubeDescSignatureUpdate.class);
-
 public CubeDescSignatureUpdate(String[] cubes) {
 config = KylinConfig.getInstanceFromEnv();
 store = ResourceStore.getStore(config);
 cubeNames = cubes;
 }
 
-public void update() {
-logger.info("Reloading Cube Metadata from store: " + 
store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
-CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
-List cubeDescs;
-if (ArrayUtils.isEmpty(cubeNames)) {
-cubeDescs = cubeDescManager.listAllDesc();
-} else {
-String[] names = cubeNames[0].split(",");
-if (ArrayUtils.isEmpty(names))
-return;
-cubeDescs = Lists.newArrayListWithCapacity(names.length);
-for (String name : names) {
-cubeDescs.add(cubeDescManager.getCubeDesc(name));
-}
-}
-for (CubeDesc cubeDesc : cubeDescs) {
-updateCubeDesc(cubeDesc);
-}
-}
-
 public static void main(String args[]) {
-if (args != null && args.length != 0 && args.length != 1) {
+if (args != null && args.length > 1) {
 System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; 
e.g, cube1,cube2 ");
 return;
 }
@@ -100,11 +83,40 @@ public class CubeDescSignatureUpdate {
 
logger.info("=");
 }
 
+public void update() {
+logger.info("Reloading Cube Metadata from store: " + 
store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+List cubeDescs;
+if (ArrayUtils.isEmpty(cubeNames)) {
+cubeDescs = cubeDescManager.listAllDesc();
+} else {
+String[] names = cubeNames[0].split(",");
+if (ArrayUtils.isEmpty(names))
+return;
+cubeDescs = Lists.newArrayListWithCapacity(names.length);
+for (String name : names) {
+cubeDescs.add(cubeDescManager.getCubeDesc(name));
+}
+}
+for (CubeDesc cubeDesc : cubeDescs) {
+

[09/10] kylin git commit: KYLIN-1190 Make memory budget per query configurable

2015-12-04 Thread mahongbin

KYLIN-1190 Make memory budget per query configurable

Signed-off-by: honma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e8e2d27a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e8e2d27a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e8e2d27a

Branch: refs/heads/2.x-staging
Commit: e8e2d27ab82d6565615a27d4a441f019a5b5aafa
Parents: 6b20569
Author: lidongsjtu 
Authored: Thu Dec 3 09:50:19 2015 +0800
Committer: honma 
Committed: Fri Dec 4 16:29:07 2015 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfig.java  |  4 
 .../kylin/storage/hbase/cube/v1/CubeStorageQuery.java   | 12 
 .../kylin/storage/hbase/cube/v2/CubeStorageQuery.java   | 11 +++
 3 files changed, 19 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index b08df5f..d2a94c5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -492,6 +492,10 @@ public class KylinConfig implements Serializable {
 return 
Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", 
String.valueOf(10 * 1024)));
 }
 
+public long getQueryMemBudget() {
+return Long.parseLong(this.getOptional("kylin.query.mem.budget", 
String.valueOf(3L * 1024 * 1024 * 1024)));
+}
+
 public boolean isQuerySecureEnabled() {
 return 
Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 4d34943..2fa0490 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -82,7 +82,6 @@ public class CubeStorageQuery implements 
ICachableStorageQuery {
 private static final Logger logger = 
LoggerFactory.getLogger(CubeStorageQuery.class);
 
 private static final int MERGE_KEYRANGE_THRESHOLD = 100;
-private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; 
// 3G
 
 private final CubeInstance cubeInstance;
 private final CubeDesc cubeDesc;
@@ -732,8 +731,13 @@ public class CubeStorageQuery implements 
ICachableStorageQuery {
 }
 }
 
-long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
-context.setThreshold((int) rowEst);
+long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / 
rowSizeEst;
+if (rowEst > 0) {
+logger.info("Memory budget is set to: " + rowEst);
+context.setThreshold((int) rowEst);
+} else {
+logger.info("Memory budget is not set.");
+}
 }
 
 private void setLimit(TupleFilter filter, StorageContext context) {
@@ -760,7 +764,7 @@ public class CubeStorageQuery implements 
ICachableStorageQuery {
 topnLiteralCol = func.getTopNLiteralColumn();
 }
 }
-
+
 // if TopN is not involved
 if (topnFunc == null)
 return;

http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 0c8c3bd..59ed99b 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -41,8 +41,6 @@ public class CubeStorageQuery implements 
ICachableStorageQuery {
 
 private static final Logger logger = 
LoggerFactory.getLogger(CubeStorageQuery.class);
 
-private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; 
// 3G
-
 private final CubeInstance cubeInstance;
 private final CubeDesc cubeDesc;
 
@@ -362,8

[6/7] kylin git commit: KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming joint semantic

2015-12-04 Thread mahongbin

KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming 
joint semantic


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07537a9a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07537a9a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07537a9a

Branch: refs/heads/2.x-staging
Commit: 07537a9aa6c6d6ed37fb039bee5b3c2eabd534d8
Parents: af6b11a
Author: honma 
Authored: Fri Nov 27 16:47:28 2015 +0800
Committer: honma 
Committed: Fri Dec 4 17:08:01 2015 +0800

--
 .../kylin/job/dataGen/FactTableGenerator.java   |   7 +-
 .../persistence/RootPersistentEntity.java   |   2 +
 .../org/apache/kylin/common/util/BasicTest.java |   2 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   6 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java| 329 +++-
 .../org/apache/kylin/cube/cuboid/CuboidCLI.java |  81 +-
 .../kylin/cube/cuboid/CuboidScheduler.java  | 256 +++---
 .../gridtable/CuboidToGridTableMapping.java |  10 +-
 .../cube/inmemcubing/InMemCubeBuilderUtils.java |   6 +-
 .../org/apache/kylin/cube/kv/RowConstants.java  |   2 +
 .../kylin/cube/model/AggregationGroup.java  | 280 +++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 187 ++---
 .../apache/kylin/cube/model/DimensionDesc.java  | 121 +--
 .../kylin/cube/model/HBaseMappingDesc.java  |   2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  73 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java | 229 +
 .../org/apache/kylin/cube/model/SelectRule.java |  30 +
 .../kylin/cube/model/v2/CubeBuildTypeEnum.java  |  39 +
 .../apache/kylin/cube/model/v2/CubeDesc.java| 826 +++
 .../kylin/cube/model/v2/DimensionDesc.java  | 239 ++
 .../kylin/cube/model/v2/HBaseColumnDesc.java| 138 
 .../cube/model/v2/HBaseColumnFamilyDesc.java|  58 ++
 .../kylin/cube/model/v2/HBaseMappingDesc.java   |  96 +++
 .../kylin/cube/model/v2/HierarchyDesc.java  |  68 ++
 .../kylin/cube/model/v2/RowKeyColDesc.java  |  92 +++
 .../apache/kylin/cube/model/v2/RowKeyDesc.java  | 295 +++
 .../model/validation/CubeMetadataValidator.java |   9 +-
 .../cube/model/validation/ValidateContext.java  |   5 +-
 .../validation/rule/AggregationGroupRule.java   | 185 +
 .../rule/AggregationGroupSizeRule.java  |  66 --
 .../rule/IKylinValidationConstants.java |   6 +-
 .../validation/rule/MandatoryColumnRule.java|  75 --
 .../model/validation/rule/RowKeyAttrRule.java   |  27 +-
 .../cube/upgrade/V2/CubeDescUpgraderV2.java | 287 +++
 .../cube/upgrade/V2/CubeMetadataUpgradeV2.java  | 177 
 .../org/apache/kylin/cube/util/CubingUtils.java |  30 +-
 .../kylin/cube/AggregationGroupRuleTest.java| 127 +++
 .../cube/AggregationGroupSizeRuleTest.java  | 101 ---
 .../org/apache/kylin/cube/CubeManagerTest.java  |   2 +-
 .../kylin/cube/DictionaryManagerTest.java   |   4 +-
 .../kylin/cube/MandatoryColumnRuleTest.java |  57 --
 .../apache/kylin/cube/RowKeyAttrRuleTest.java   |  43 +-
 .../kylin/cube/cuboid/CuboidSchedulerTest.java  | 155 ++--
 .../apache/kylin/cube/cuboid/CuboidTest.java|  10 +-
 .../src/test/resources/data/TEST1_desc.json | 183 
 .../src/test/resources/data/TEST2_desc.json | 168 
 .../src/test/resources/data/TEST3_desc.json | 182 
 .../apache/kylin/dict/DictionaryManager.java|  10 +-
 .../kylin/job/manager/ExecutableManager.java|   4 +-
 .../kylin/metadata/model/MeasureDesc.java   |  16 +-
 .../kylin/engine/mr/BatchCubingJobBuilder.java  |   2 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java |   2 +-
 .../mr/steps/FactDistinctHiveColumnsMapper.java |   3 +-
 .../mr/steps/MapContextGTRecordWriter.java  |   2 +-
 .../mr/steps/MergeCuboidFromStorageMapper.java  |   2 +-
 .../engine/mr/steps/MergeCuboidMapper.java  |   2 +-
 .../engine/mr/steps/MergeDictionaryStep.java|   2 +-
 .../kylin/engine/mr/steps/CubeSamplingTest.java |   3 +-
 .../apache/kylin/engine/spark/SparkCubing.java  |   5 +-
 .../spark/cube/DefaultTupleConverter.java   |   3 +-
 .../apache/kylin/rest/service/CubeService.java  |   4 +-
 .../rest/controller/CubeControllerTest.java |   7 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |   2 +-
 .../hbase/cube/v1/CubeTupleConverter.java   |   6 +-
 .../storage/hbase/cube/v2/CubeStorageQuery.java |   2 +-
 .../hbase/cube/v2/CubeTupleConverter.java   |   3 +-
 .../kylin/storage/hbase/steps/BulkLoadJob.java  |   2 +-
 .../storage/hbase/steps/CubeHFileMapper.java|   2 +-
 .../storage/hbase/steps/CubeHTableUtil.java |   2 +-
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   2 +-
 .../storage/hbase/steps/HBaseMROutput2.java |   4 +-
 .../hbase/steps/HBaseMROutput2Transition.java   |   2 +-

[2/7] kylin git commit: KYLIN-242 refine cube desc to fit new aggregation group design

2015-12-04 Thread mahongbin

KYLIN-242 refine cube desc to fit new aggregation group design


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af6b11a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af6b11a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af6b11a9

Branch: refs/heads/2.x-staging
Commit: af6b11a97c74eb45aba62fa65494bab98d33541c
Parents: 91b1f92
Author: honma 
Authored: Mon Nov 30 11:17:14 2015 +0800
Committer: honma 
Committed: Fri Dec 4 17:07:50 2015 +0800

--
 .../localmeta/cube_desc/test_kylin_cube_without_slr_desc.json | 3 +--
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 3 +--
 .../localmeta/cube_desc/test_streaming_table_cube_desc.json   | 3 ++-
 3 files changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/af6b11a9/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 1e007c3..31c54a8 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -245,8 +245,7 @@
 "leaf_categ_id",
 "meta_categ_name",
 "categ_lvl3_name",
-"categ_lvl2_name",
-"lstg_format_name"
+"categ_lvl2_name"
   ]
 ]
   },

http://git-wip-us.apache.org/repos/asf/kylin/blob/af6b11a9/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 73a58f0..80ea23a 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -245,8 +245,7 @@
 "leaf_categ_id",
 "meta_categ_name",
 "categ_lvl3_name",
-"categ_lvl2_name",
-"lstg_format_name"
+"categ_lvl2_name"
   ]
 ]
   },

http://git-wip-us.apache.org/repos/asf/kylin/blob/af6b11a9/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
index c9a6536..6f9b3a7 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_streaming_table_cube_desc.json
@@ -83,7 +83,7 @@
 "column": "DAY_START",
 "length": 0,
 "dictionary": "true",
-"mandatory": true
+"mandatory": false
   },
   {
 "column": "HOUR_START",
@@ -112,6 +112,7 @@
 ],
 "aggregation_groups": [
   [
+"DAY_START",
 "HOUR_START",
 "MINUTE_START",
 "SITE",

[3/7] kylin git commit: KYLIN-242 refactor cuboid and cuboid scheduer to support white list by addming joint semantic

2015-12-04 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 981e19a..2a1304a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -223,11 +223,11 @@ public class DictionaryManager {
 }
 }
 
-public DictionaryInfo buildDictionary(DataModelDesc model, String dict, 
TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws 
IOException {
+public DictionaryInfo buildDictionary(DataModelDesc model, boolean 
usingDict, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) 
throws IOException {
 
 logger.info("building dictionary for " + col);
 
-TblColRef srcCol = decideSourceData(model, dict, col);
+TblColRef srcCol = decideSourceData(model, usingDict, col);
 String srcTable = srcCol.getTable();
 String srcColName = srcCol.getName();
 int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
@@ -259,9 +259,9 @@ public class DictionaryManager {
 /**
  * Decide a dictionary's source data, leverage PK-FK relationship.
  */
-public TblColRef decideSourceData(DataModelDesc model, String dict, 
TblColRef col) throws IOException {
+public TblColRef decideSourceData(DataModelDesc model, boolean usingDict, 
TblColRef col) throws IOException {
 // Note FK on fact table is supported by scan the related PK on lookup 
table
-if ("true".equals(dict) || "string".equals(dict) || 
"number".equals(dict) || "any".equals(dict)) {
+if (usingDict) {
 // FK on fact table and join type is inner, use PK from lookup 
instead
 if (model.isFactTable(col.getTable())) {
 TblColRef pkCol = model.findPKByFK(col, "inner");
@@ -270,7 +270,7 @@ public class DictionaryManager {
 }
 return col;
 } else
-throw new IllegalArgumentException("Unknown dictionary value: " + 
dict);
+throw new IllegalArgumentException("Not using Dictionary ");
 }
 
 private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
index ff92a50..d73fd26 100644
--- a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
@@ -299,6 +299,7 @@ public class ExecutableManager {
 
 private static AbstractExecutable parseTo(ExecutablePO executablePO) {
 if (executablePO == null) {
+logger.warn("executablePO is null");
 return null;
 }
 String type = executablePO.getType();
@@ -318,8 +319,7 @@ public class ExecutableManager {
 }
 return result;
 } catch (ReflectiveOperationException e) {
-logger.error("cannot parse this job:" + executablePO.getId(), e);
-return null;
+throw new IllegalStateException("cannot parse this job:" + 
executablePO.getId(), e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/07537a9a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 618d25a..5d65771 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -31,8 +31,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
 public class MeasureDesc {
 
-@JsonProperty("id")
-private int id;
+   
 @JsonProperty("name")
 private String name;
 @JsonProperty("function")
@@ -49,13 +48,7 @@ public class MeasureDesc {
 }
 }
 
-public int getId() {
-return id;
-}
-
-public void setId(int id) {
-this.id = id;
-}
+   
 
 public String getName() {
 return name;
@@ -103,11 +96,6 @@ public class MeasureDesc

< 4 5 6 7 8 9 10 11 12 13 >

801 - 900 of 1363 matches

Mail list logo