kylin git commit: minor code reformat

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 0453d3daa -> 4f466a07b


minor code reformat


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f466a07
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f466a07
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f466a07

Branch: refs/heads/master
Commit: 4f466a07bfb5438a1c573699f0b01cd206b0423a
Parents: 0453d3d
Author: Li Yang 
Authored: Mon Dec 26 15:59:03 2016 +0800
Committer: Li Yang 
Committed: Mon Dec 26 15:59:03 2016 +0800

--
 .../java/org/apache/kylin/provision/ProvisionWithoutBuild.java | 2 +-
 .../main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4f466a07/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
index 71bb0ca..039d583 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/provision/ProvisionWithoutBuild.java
@@ -24,6 +24,6 @@ public class ProvisionWithoutBuild extends 
BuildCubeWithEngine {
 public static void main(String[] args) throws Exception {
 beforeClass();
 DeployUtil.deployMetadata();
-
DeployUtil.prepareTestDataForNormalCubes("test_kylin_cube_with_slr_empty");
+DeployUtil.prepareTestDataForNormalCubes("ci_left_join_model");
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4f466a07/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index 93f06dd..132e65a 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -163,6 +163,6 @@ public class OLAPSchemaFactory implements SchemaFactory {
 }
 index++;
 }
-out.write("  ]\n");
+out.write("]\n");
 }
 }

kylin git commit: KYLIN-2301 various fixes to support NOT IN subquery

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master ed8b1683a -> 0453d3daa


KYLIN-2301 various fixes to support NOT IN subquery


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0453d3da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0453d3da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0453d3da

Branch: refs/heads/master
Commit: 0453d3daada2e32f79954dfd12ac9f19062e825c
Parents: ed8b168
Author: Li Yang 
Authored: Fri Dec 23 08:17:28 2016 +0800
Committer: Li Yang 
Committed: Mon Dec 26 15:26:25 2016 +0800

--
 .../kylin/metadata/filter/LogicalTupleFilter.java  | 13 -
 .../org/apache/kylin/metadata/filter/TupleFilter.java  |  7 ++-
 .../org/apache/kylin/metadata/model/TblColRef.java |  7 ++-
 .../java/org/apache/kylin/query/ITKylinQueryTest.java  |  2 +-
 .../src/test/resources/query/sql_subquery/query13.sql  |  7 +++
 .../apache/kylin/query/relnode/OLAPAggregateRel.java   | 11 +--
 .../org/apache/kylin/query/relnode/OLAPJoinRel.java|  8 +++-
 7 files changed, 32 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 1744309..373acdb 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -47,19 +47,6 @@ public class LogicalTupleFilter extends TupleFilter {
 return cloneTuple;
 }
 
-//private TupleFilter reverseNestedNots(TupleFilter filter, int depth) 
{
-//if ((filter instanceof LogicalTupleFilter) && (filter.operator 
== FilterOperatorEnum.NOT)) {
-//assert (filter.children.size() == 1);
-//return reverseNestedNots(filter.children.get(0), depth + 1);
-//}
-//
-//if (depth % 2 == 1) {
-//return filter;
-//} else {
-//return filter.reverse();
-//}
-//}
-
 @Override
 public TupleFilter reverse() {
 switch (operator) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 285172c..31de50b 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -27,6 +27,8 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Maps;
 
@@ -37,6 +39,8 @@ import com.google.common.collect.Maps;
  */
 public abstract class TupleFilter {
 
+static final Logger logger = LoggerFactory.getLogger(TupleFilter.class);
+
 public enum FilterOperatorEnum {
 EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), 
IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), 
DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), EVAL_FUNC(37), 
UNSUPPORTED(38);
 
@@ -112,7 +116,8 @@ public abstract class TupleFilter {
 }
 
 public TupleFilter reverse() {
-throw new UnsupportedOperationException();
+logger.warn("Cannot reverse " + this + ", loosen the filter to true");
+return ConstantTupleFilter.TRUE;
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/0453d3da/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index fd0224f..88abb9d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -27,7 +27,7 @@ import org.apache.kylin.metadata.datatype.DataType;
 
 /**
  */
-@SuppressWarnings("serial")
+@SuppressWarnings({ "serial", "deprecation" })
 public class 

kylin git commit: KYLIN-2314 more CI fixes

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 73fbbb90e -> ed8b1683a


KYLIN-2314 more CI fixes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed8b1683
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed8b1683
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed8b1683

Branch: refs/heads/master
Commit: ed8b1683a35488ffb8dd222c7bbe0fa84600aa5d
Parents: 73fbbb9
Author: Yang Li 
Authored: Sat Dec 24 21:09:21 2016 +0800
Committer: Yang Li 
Committed: Mon Dec 26 07:25:54 2016 +0800

--
 examples/test_case_data/localmeta/project/default.json  | 9 -
 .../main/java/org/apache/kylin/query/schema/OLAPTable.java  | 2 +-
 .../kylin/source/kafka/config/KafkaConsumerProperties.java  | 8 +++-
 3 files changed, 12 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/examples/test_case_data/localmeta/project/default.json
--
diff --git a/examples/test_case_data/localmeta/project/default.json 
b/examples/test_case_data/localmeta/project/default.json
index 1ecc7cc..75d7ec9 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -31,11 +31,18 @@
   "tables": [
 "DEFAULT.TEST_KYLIN_FACT",
 "DEFAULT.TEST_ORDER",
+"DEFAULT.TEST_ACCOUNT",
+"DEFAULT.TEST_COUNTRY",
 "EDW.TEST_CAL_DT",
 "DEFAULT.TEST_CATEGORY_GROUPINGS",
 "EDW.TEST_SITES",
 "EDW.TEST_SELLER_TYPE_DIM",
-"DEFAULT.STREAMING_TABLE"
+"DEFAULT.STREAMING_TABLE",
+"SSB.CUSTOMER",
+"SSB.DATES",
+"SSB.PART",
+"SSB.SUPPLIER",
+"SSB.V_LINEORDER"
   ],
   "models": [
 "ssb",

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index c33c4ed..85996f4 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -203,7 +203,7 @@ public class OLAPTable extends AbstractQueryableTable 
implements TranslatableTab
 DataModelDesc model = realization.getModel();
 for (String metricColumn : model.getMetrics()) {
 TblColRef col = model.findColumn(metricColumn);
-if (col.getType().isIntegerFamily() && 
!col.getType().isBigInt())
+if (col.getTable().equals(sourceTable.getIdentity()) && 
col.getType().isIntegerFamily() && !col.getType().isBigInt())
 updateColumns.add(col.getColumnDesc());
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ed8b1683/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
index 8f0dd42..5b3dd87 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
@@ -101,8 +101,8 @@ public class KafkaConsumerProperties {
 private Properties loadKafkaConsumerProperties() {
 File propFile = getKafkaConsumerFile();
 if (propFile == null || !propFile.exists()) {
-logger.error("fail to locate " + KAFKA_CONSUMER_FILE);
-throw new RuntimeException("fail to locate " + 
KAFKA_CONSUMER_FILE);
+logger.warn("fail to locate " + KAFKA_CONSUMER_FILE + ", use empty 
kafka consumer properties");
+return new Properties();
 }
 Properties properties = new Properties();
 try {
@@ -115,7 +115,6 @@ public class KafkaConsumerProperties {
 File propOverrideFile = new File(propFile.getParentFile(), 
propFile.getName() + ".override");
 if (propOverrideFile.exists()) {
 FileInputStream ois = new FileInputStream(propOverrideFile);
-Properties propOverride = new Properties();
 Configuration oconf = new Configuration();
 oconf.addResource(ois);
 properties.putAll(extractKafkaConfigToProperties(oconf));
@@ -134,7 +133,6 @@ public class KafkaConsumerProperties {
 }
 
 private File getKafkaConsumerFile() {
-

[5/6] kylin git commit: Rename sparkcubingv3 class name and add unpersistent

[shaofengshi]

Rename sparkcubingv3 class name and add unpersistent


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a648fbc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a648fbc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a648fbc

Branch: refs/heads/sparkcubing-rebase
Commit: 0a648fbc3ce2dfe142179205fc972764b5d1ab40
Parents: f4db4b5
Author: shaofengshi 
Authored: Sun Dec 25 20:28:43 2016 +0800
Committer: shaofengshi 
Committed: Sun Dec 25 20:28:43 2016 +0800

--
 .../spark/SparkBatchCubingJobBuilder2.java  |  12 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  | 360 +++
 .../kylin/engine/spark/SparkCubingV3.java   | 354 --
 3 files changed, 366 insertions(+), 360 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0a648fbc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 57b6432..9532d31 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -48,12 +48,12 @@ public class SparkBatchCubingJobBuilder2 extends 
BatchCubingJobBuilder2 {
 protected AbstractExecutable createInMemCubingStep(String jobId, String 
cuboidRootPath) {
 IJoinedFlatTableDesc flatTableDesc = 
EngineFactory.getJoinedFlatTableDesc(seg);
 final SparkExecutable sparkExecutable = new SparkExecutable();
-sparkExecutable.setClassName(SparkCubingV3.class.getName());
-sparkExecutable.setParam(SparkCubingV3.OPTION_CUBE_NAME.getOpt(), 
seg.getRealization().getName());
-sparkExecutable.setParam(SparkCubingV3.OPTION_SEGMENT_ID.getOpt(), 
seg.getUuid());
-sparkExecutable.setParam(SparkCubingV3.OPTION_INPUT_PATH.getOpt(), 
flatTableDesc.getTableName());
-sparkExecutable.setParam(SparkCubingV3.OPTION_CONF_PATH.getOpt(), 
"/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox/"); 
//FIXME
-sparkExecutable.setParam(SparkCubingV3.OPTION_OUTPUT_PATH.getOpt(), 
cuboidRootPath);
+sparkExecutable.setClassName(SparkCubingByLayer.class.getName());
+sparkExecutable.setParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt(), 
seg.getRealization().getName());
+
sparkExecutable.setParam(SparkCubingByLayer.OPTION_SEGMENT_ID.getOpt(), 
seg.getUuid());
+
sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_PATH.getOpt(), 
flatTableDesc.getTableName());
+sparkExecutable.setParam(SparkCubingByLayer.OPTION_CONF_PATH.getOpt(), 
"/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox/"); 
//FIXME
+
sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), 
cuboidRootPath);
 
 StringBuilder jars = new StringBuilder();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/0a648fbc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
new file mode 100644
index 000..7467e86
--- /dev/null
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.engine.spark;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import 

[2/6] kylin git commit: spark cubing init commit

[shaofengshi]

http://git-wip-us.apache.org/repos/asf/kylin/blob/78e6cd5b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 4d89e1a..ce5888d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -18,27 +18,27 @@
 
 package org.apache.kylin.metadata.model;
 
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.measure.MeasureTypeFactory;
-import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
 
 /**
  */
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
-public class FunctionDesc {
+public class FunctionDesc implements Serializable {
 
 public static FunctionDesc newInstance(String expression, ParameterDesc 
param, String returnType) {
 FunctionDesc r = new FunctionDesc();

http://git-wip-us.apache.org/repos/asf/kylin/blob/78e6cd5b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 6489244..dd1500b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -18,17 +18,18 @@
 
 package org.apache.kylin.metadata.model;
 
-import java.util.Arrays;
-
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
 
+import java.io.Serializable;
+import java.util.Arrays;
+
 /**
  */
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
-public class JoinDesc {
+public class JoinDesc implements Serializable {
 
 // inner, left, right, outer...
 @JsonProperty("type")

http://git-wip-us.apache.org/repos/asf/kylin/blob/78e6cd5b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
index 5d0409a..51e5787 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
@@ -25,8 +25,10 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import java.io.Serializable;
+
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = 
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = 
Visibility.NONE)
-public class JoinTableDesc {
+public class JoinTableDesc implements Serializable {
 
 @JsonProperty("table")
 private String table;

http://git-wip-us.apache.org/repos/asf/kylin/blob/78e6cd5b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index a0b267d..c132d0e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -18,15 

[3/6] kylin git commit: spark cubing init commit

[shaofengshi]

spark cubing init commit


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78e6cd5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78e6cd5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78e6cd5b

Branch: refs/heads/sparkcubing-rebase
Commit: 78e6cd5b771d96744e379e66913277e54632552b
Parents: 73fbbb9
Author: shaofengshi 
Authored: Sun Dec 25 15:59:16 2016 +0800
Committer: shaofengshi 
Committed: Sun Dec 25 19:17:32 2016 +0800

--
 assembly/pom.xml|   5 +-
 .../main/config/assemblies/source-assembly.xml  |   3 -
 .../java/org/apache/kylin/job/DeployUtil.java   |   1 -
 build/conf/kylin-spark-conf.properties  |  27 ++
 .../apache/kylin/common/KylinConfigBase.java|  38 +-
 .../kylin/common/persistence/ResourceStore.java |  35 ++
 .../org/apache/kylin/common/util/Array.java |   2 +-
 .../apache/kylin/common/util/SplittedBytes.java |   2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   2 +-
 .../kylin/cube/common/RowKeySplitter.java   |  12 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java|  27 +-
 .../kylin/cube/cuboid/CuboidScheduler.java  |  15 +-
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|   6 +-
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |   7 +-
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|   2 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java |  19 +-
 .../kylin/cube/model/AggregationGroup.java  |   4 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |   2 +-
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   6 +-
 .../apache/kylin/cube/model/DictionaryDesc.java |   2 +-
 .../apache/kylin/cube/model/DimensionDesc.java  |   2 +-
 .../kylin/cube/model/HBaseColumnDesc.java   |  11 +-
 .../kylin/cube/model/HBaseColumnFamilyDesc.java |   9 +-
 .../kylin/cube/model/HBaseMappingDesc.java  |  15 +-
 .../apache/kylin/cube/model/HierarchyDesc.java  |   2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |   2 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java |  17 +-
 .../org/apache/kylin/cube/model/SelectRule.java |   2 +-
 .../dict/NumberDictionaryForestBuilder.java |   2 +-
 .../apache/kylin/dict/StringBytesConverter.java |   2 +-
 .../kylin/job/execution/ExecutableManager.java  |  21 +-
 .../kylin/dimension/AbstractDateDimEnc.java |  12 +-
 .../apache/kylin/dimension/BooleanDimEnc.java   |   4 +-
 .../kylin/dimension/DictionaryDimEnc.java   |  21 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  |   4 +-
 .../kylin/dimension/FixedLenHexDimEnc.java  |   4 +-
 .../org/apache/kylin/dimension/IntDimEnc.java   |   4 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   4 +-
 .../kylin/dimension/OneMoreByteVLongDimEnc.java |  14 +-
 .../kylin/measure/BufferedMeasureCodec.java |  10 +-
 .../org/apache/kylin/measure/MeasureCodec.java  |   8 +-
 .../apache/kylin/measure/MeasureIngester.java   |  12 +-
 .../org/apache/kylin/measure/MeasureType.java   |  12 +-
 .../kylin/measure/basic/DoubleIngester.java |   5 +
 .../measure/basic/DoubleSumAggregator.java  |   3 +-
 .../kylin/measure/basic/LongIngester.java   |   5 +
 .../kylin/measure/basic/LongSumAggregator.java  |   3 +-
 .../kylin/measure/bitmap/BitmapCounter.java |   8 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |   5 +
 .../kylin/measure/hllc/HLLCMeasureType.java |   5 +
 .../kylin/measure/hllc/HLLCSerializer.java  |  11 +-
 .../apache/kylin/measure/raw/RawSerializer.java |   4 +-
 .../apache/kylin/measure/topn/TopNCounter.java  |   2 +-
 .../kylin/measure/topn/TopNMeasureType.java |   4 +-
 .../metadata/datatype/BigDecimalSerializer.java |   2 +-
 .../metadata/datatype/BooleanSerializer.java|   9 +-
 .../metadata/datatype/DataTypeSerializer.java   |  17 +-
 .../metadata/datatype/DateTimeSerializer.java   |   9 +-
 .../metadata/datatype/DoubleSerializer.java |   5 +-
 .../kylin/metadata/datatype/Int4Serializer.java |   9 +-
 .../metadata/datatype/Long8Serializer.java  |   9 +-
 .../kylin/metadata/datatype/LongSerializer.java |   9 +-
 .../kylin/metadata/model/FunctionDesc.java  |  22 +-
 .../apache/kylin/metadata/model/JoinDesc.java   |   7 +-
 .../kylin/metadata/model/JoinTableDesc.java |   4 +-
 .../apache/kylin/metadata/model/JoinsTree.java  |   9 +-
 .../kylin/metadata/model/MeasureDesc.java   |   9 +-
 .../metadata/model/ModelDimensionDesc.java  |  10 +-
 .../kylin/metadata/model/ParameterDesc.java |  13 +-
 .../kylin/metadata/model/PartitionDesc.java |   6 +-
 .../apache/kylin/metadata/model/TableRef.java   |  17 +-
 .../kylin/engine/mr/BatchCubingJobBuilder2.java |   9 +-
 .../kylin/engine/mr/JobBuilderSupport.java  |  12 +-
 .../engine/mr/common/BaseCuboidBuilder.java | 175 +
 

[4/6] kylin git commit: rebase with master

[shaofengshi]

rebase with master


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f4db4b59
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f4db4b59
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f4db4b59

Branch: refs/heads/sparkcubing-rebase
Commit: f4db4b59feeb5ce633809085b7a77b296ac404f8
Parents: 78e6cd5
Author: shaofengshi 
Authored: Sun Dec 25 19:21:02 2016 +0800
Committer: shaofengshi 
Committed: Sun Dec 25 19:21:02 2016 +0800

--
 .../main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f4db4b59/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index b2d2ddc..de36b08 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -108,7 +108,7 @@ public class HLLCMeasureType extends 
MeasureType {
 
 @Override
 public void reset() {
-current = new HyperLogLogPlusCounter(dataType.getPrecision());
+current = new HLLCounter(dataType.getPrecision());
 }
 };
 }

[1/6] kylin git commit: spark cubing init commit

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/sparkcubing-rebase [created] 55ba39afb


http://git-wip-us.apache.org/repos/asf/kylin/blob/78e6cd5b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..763b537 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.collect.Sets;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -56,8 +57,6 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Sets;
-
 public class HiveMRInput implements IMRInput {
 
 public static String getTableNameForHCat(TableDesc table) {
@@ -74,7 +73,7 @@ public class HiveMRInput implements IMRInput {
 public IMRTableInputFormat getTableInputFormat(TableDesc table) {
 return new HiveTableInputFormat(getTableNameForHCat(table));
 }
-
+
 @Override
 public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) {
 return new IMRBatchMergeInputSide() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/78e6cd5b/tool/pom.xml
--
diff --git a/tool/pom.xml b/tool/pom.xml
index 8eddec4..b577ae9 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -40,6 +40,10 @@
 
 
 org.apache.kylin
+kylin-engine-spark
+
+
+org.apache.kylin
 kylin-source-kafka
 
 

[6/6] kylin git commit: Use java built-in class to replace LongMutable/DoubleMutable;

[shaofengshi]

Use java built-in class to replace LongMutable/DoubleMutable;


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55ba39af
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55ba39af
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55ba39af

Branch: refs/heads/sparkcubing-rebase
Commit: 55ba39afb6ded85e5f8d82df2ba6828dc4b68819
Parents: 0a648fb
Author: shaofengshi 
Authored: Sun Dec 25 22:48:43 2016 +0800
Committer: shaofengshi 
Committed: Sun Dec 25 22:48:43 2016 +0800

--
 build/conf/kylin.properties | 11 ++
 .../apache/kylin/common/KylinConfigBase.java| 17 +++-
 .../cube/inmemcubing/InMemCubeBuilder.java  |  2 +
 .../apache/kylin/gridtable/UnitTestSupport.java | 41 ++--
 .../gridtable/AggregationCacheMemSizeTest.java  |  6 +--
 .../gridtable/AggregationCacheSpillTest.java|  5 +--
 .../kylin/gridtable/SimpleGridTableTest.java| 11 +++---
 .../gridtable/SimpleInvertedIndexTest.java  |  2 +-
 .../metadata/measure/MeasureCodecTest.java  |  6 +--
 .../kylin/measure/basic/DoubleIngester.java | 16 +++-
 .../measure/basic/DoubleMaxAggregator.java  | 15 ---
 .../measure/basic/DoubleMinAggregator.java  | 15 ---
 .../measure/basic/DoubleSumAggregator.java  | 18 +
 .../kylin/measure/basic/LongIngester.java   | 16 +++-
 .../kylin/measure/basic/LongMaxAggregator.java  | 15 ---
 .../kylin/measure/basic/LongMinAggregator.java  | 15 ---
 .../kylin/measure/basic/LongSumAggregator.java  | 18 +
 .../metadata/datatype/BooleanSerializer.java| 29 --
 .../metadata/datatype/DateTimeSerializer.java   | 25 
 .../metadata/datatype/DoubleSerializer.java | 25 
 .../kylin/metadata/datatype/Int4Serializer.java | 25 
 .../metadata/datatype/Long8Serializer.java  | 24 
 .../kylin/metadata/datatype/LongSerializer.java | 25 
 .../measure/AggregatorMemEstimateTest.java  |  4 +-
 .../storage/gtrecord/DictGridTableTest.java | 41 ++--
 .../kylin/engine/mr/steps/CubeReducerTest.java  |  3 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  | 32 +--
 .../test_case_data/sandbox/kylin.properties |  6 +--
 .../hbase/steps/RowValueDecoderTest.java|  7 ++--
 29 files changed, 199 insertions(+), 276 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 131a725..dabe113 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -128,6 +128,17 @@ kylin.engine.mr.max-reducer-number=500
 kylin.engine.mr.mapper-input-rows=100
 
 
+### Spark Engine ###
+
+# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run 
spark-submit
+kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+
+# Spark install home, default be $KYLIN_HOME/spark/
+#kylin.engine.spark.spark-home=
+
+# Spark job submission properties file, default be 
$KYLIN_HOME/conf/kylin-spark-conf.properties
+#kylin.engine.spark.properties-file=
+
 ### CUBE | DICTIONARY ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index c32550f..5bec9b3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -714,7 +714,20 @@ abstract public class KylinConfigBase implements 
Serializable {
 // 

 
 public String getSparkHome() {
-return getRequired("kylin.engine.spark.spark-home");
+String sparkHome = getOptional("kylin.engine.spark.spark-home", 
"spark");
+File f = new File(sparkHome);
+if (f.exists()) {
+return f.getAbsolutePath();
+} else {
+String home = getKylinHome();
+f = new File(home, sparkHome);
+if (f.exists()) {
+return f.getAbsolutePath();
+}
+}
+
+throw new IllegalArgumentException("Spark home '" + sparkHome + "' 
does not exist, check 'kylin.engine.spark.spark-home' in kylin.properties");
+
 }
 
 public String getSparkHadoopConfDir() {
@@ -734,7 +747,7 @@ abstract 

[1/8] kylin git commit: KYLIN-2318 query cache is not working [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 c0da9e250 -> 36f64d6a1 (forced update)


KYLIN-2318 query cache is not working


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8596ccf2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8596ccf2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8596ccf2

Branch: refs/heads/yang22-hbase102
Commit: 8596ccf262241826cb60baf3a9a3999c39ae0c66
Parents: da5e5d6
Author: Hongbin Ma 
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 15:31:22 2016 +0800

--
 .../org/apache/kylin/common/QueryContext.java   | 61 
 .../kylin/common/debug/BackdoorToggles.java | 11 ++--
 .../apache/kylin/rest/request/SQLRequest.java   | 23 +---
 .../apache/kylin/rest/service/QueryService.java | 23 
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  4 +-
 5 files changed, 93 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java 
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for 
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new 
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.Pair;
 
 /**
+ * BackdoorToggles and QueryContext are similar because they're both hosting 
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work 
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ * 
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to 
SQLRequest
  */
 public class BackdoorToggles {
 
@@ -65,10 +70,6 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-  

[6/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[3/8] kylin git commit: strenthen storage visit deadline calculation

[lidong]

strenthen storage visit deadline calculation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6bb13829
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6bb13829
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6bb13829

Branch: refs/heads/yang22-hbase102
Commit: 6bb1382970f8581b3f8a666ac98829e2a501ffd6
Parents: 6e18274
Author: Hongbin Ma 
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 17:57:55 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6bb13829/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 }
 
 final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() + 
scanReq.getTimeout();
 logger.info("deadline is " + deadline);
 final long storagePushDownLimit = 
scanReq.getStoragePushDownLimit();
 

[5/8] kylin git commit: bug fix

[lidong]

bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/473f77b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/473f77b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/473f77b0

Branch: refs/heads/yang22-hbase102
Commit: 473f77b06a1554aedbfcfd4fc5b827087aadb426
Parents: 83f9dc7
Author: Hongbin Ma 
Authored: Sun Dec 25 14:51:08 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Dec 25 14:51:08 2016 +0800

--
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/473f77b0/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 408d227..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -76,14 +76,15 @@ public class OLAPEnumerator implements Enumerator 
{
 }
 convertCurrentRow(tuple);
 return true;
-} finally {
+} catch (Exception e) {
 try {
 if (cursor != null) {
 cursor.close();
 }
-} catch (Exception e) {
-logger.info("Error when closing cursor, ignore it", e);
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
 }
+throw e;
 }
 }
 

[2/8] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log

[lidong]

introduce CheckUtil to make multiple condition checks clearer in log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e18274c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e18274c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e18274c

Branch: refs/heads/yang22-hbase102
Commit: 6e18274c150e21fc952866553e0fd22139477bf1
Parents: 8596ccf
Author: Hongbin Ma 
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 16:52:19 2016 +0800

--
 .../org/apache/kylin/common/util/CheckUtil.java | 33 
 .../apache/kylin/rest/service/QueryService.java | 17 ++
 2 files changed, 44 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message, 
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.rest.service;
 
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
 import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
 long startTime = System.currentTimeMillis();
 
 SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && 
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled = 
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in 
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query 
cache disabled in BackdoorToggles");
+
 if (queryCacheEnabled) {
 sqlResponse = searchQueryInCache(sqlRequest);
 }
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
 sqlResponse.setDuration(System.currentTimeMillis() - 
startTime);
 logger.info("Stats of SQL response: isException: {}, 
duration: {}, total scan count {}", //
 String.valueOf(sqlResponse.getIsException()), 
String.valueOf(sqlResponse.getDuration()), 
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold || 
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() < 
kylinConfig.getLargeQueryThreshold())) { //don't cache too large 

[4/8] kylin git commit: close OLAPEnumerator at exception

[lidong]

close OLAPEnumerator at exception


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f9dc7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f9dc7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f9dc7a

Branch: refs/heads/yang22-hbase102
Commit: 83f9dc7a68e64c8c71dd3dea971f9f0311a5575c
Parents: 6bb1382
Author: Hongbin Ma 
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 18:19:02 2016 +0800

--
 .../kylin/query/enumerator/OLAPEnumerator.java  | 32 +---
 1 file changed, 21 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f9dc7a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..408d227 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,30 @@ public class OLAPEnumerator implements Enumerator 
{
 
 @Override
 public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
 
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
 
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} finally {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception e) {
+logger.info("Error when closing cursor, ignore it", e);
+}
 }
-convertCurrentRow(tuple);
-return true;
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {

[3/8] kylin git commit: strenthen storage visit deadline calculation

[lidong]

strenthen storage visit deadline calculation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6bb13829
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6bb13829
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6bb13829

Branch: refs/heads/yang22-cdh5.7
Commit: 6bb1382970f8581b3f8a666ac98829e2a501ffd6
Parents: 6e18274
Author: Hongbin Ma 
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 17:57:55 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6bb13829/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 }
 
 final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() + 
scanReq.getTimeout();
 logger.info("deadline is " + deadline);
 final long storagePushDownLimit = 
scanReq.getStoragePushDownLimit();
 

[5/8] kylin git commit: bug fix

[lidong]

bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/473f77b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/473f77b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/473f77b0

Branch: refs/heads/yang22-cdh5.7
Commit: 473f77b06a1554aedbfcfd4fc5b827087aadb426
Parents: 83f9dc7
Author: Hongbin Ma 
Authored: Sun Dec 25 14:51:08 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Dec 25 14:51:08 2016 +0800

--
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/473f77b0/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 408d227..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -76,14 +76,15 @@ public class OLAPEnumerator implements Enumerator 
{
 }
 convertCurrentRow(tuple);
 return true;
-} finally {
+} catch (Exception e) {
 try {
 if (cursor != null) {
 cursor.close();
 }
-} catch (Exception e) {
-logger.info("Error when closing cursor, ignore it", e);
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
 }
+throw e;
 }
 }
 

[7/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49b8d186
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49b8d186
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49b8d186

Branch: refs/heads/yang22-cdh5.7
Commit: 49b8d1865b954a4395fe5965988be9297a553454
Parents: 473f77b
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Sun Dec 25 20:13:27 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[4/8] kylin git commit: close OLAPEnumerator at exception

[lidong]

close OLAPEnumerator at exception


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f9dc7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f9dc7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f9dc7a

Branch: refs/heads/yang22-cdh5.7
Commit: 83f9dc7a68e64c8c71dd3dea971f9f0311a5575c
Parents: 6bb1382
Author: Hongbin Ma 
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 18:19:02 2016 +0800

--
 .../kylin/query/enumerator/OLAPEnumerator.java  | 32 +---
 1 file changed, 21 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f9dc7a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..408d227 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,30 @@ public class OLAPEnumerator implements Enumerator 
{
 
 @Override
 public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
 
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
 
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} finally {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception e) {
+logger.info("Error when closing cursor, ignore it", e);
+}
 }
-convertCurrentRow(tuple);
-return true;
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {

[1/8] kylin git commit: KYLIN-2318 query cache is not working [Forced Update!]

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 68fda3f72 -> d7d009339 (forced update)


KYLIN-2318 query cache is not working


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8596ccf2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8596ccf2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8596ccf2

Branch: refs/heads/yang22-cdh5.7
Commit: 8596ccf262241826cb60baf3a9a3999c39ae0c66
Parents: da5e5d6
Author: Hongbin Ma 
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 15:31:22 2016 +0800

--
 .../org/apache/kylin/common/QueryContext.java   | 61 
 .../kylin/common/debug/BackdoorToggles.java | 11 ++--
 .../apache/kylin/rest/request/SQLRequest.java   | 23 +---
 .../apache/kylin/rest/service/QueryService.java | 23 
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  4 +-
 5 files changed, 93 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java 
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for 
comparision
+ */
+public class QueryContext {
+private static final ThreadLocal> _queryContext = new 
ThreadLocal>();
+
+public final static String KEY_QUERY_ID = "QUERY_ID";
+
+public static String getQueryId() {
+return getString(KEY_QUERY_ID);
+}
+
+public static void setQueryId(String uuid) {
+setString(KEY_QUERY_ID, uuid);
+}
+
+private static void setString(String key, String value) {
+Map context = _queryContext.get();
+if (context == null) {
+Map newMap = Maps.newHashMap();
+newMap.put(key, value);
+_queryContext.set(newMap);
+} else {
+context.put(key, value);
+}
+}
+
+private static String getString(String key) {
+Map context = _queryContext.get();
+if (context == null) {
+return null;
+} else {
+return context.get(key);
+}
+}
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/8596ccf2/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.Pair;
 
 /**
+ * BackdoorToggles and QueryContext are similar because they're both hosting 
per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work 
for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ * 
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to 
SQLRequest
  */
 public class BackdoorToggles {
 
@@ -65,10 +70,6 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
-public static String getQueryId() {
-return getString(KEY_QUERY_ID);
-}

[6/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[2/8] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log

[lidong]

introduce CheckUtil to make multiple condition checks clearer in log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e18274c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e18274c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e18274c

Branch: refs/heads/yang22-cdh5.7
Commit: 6e18274c150e21fc952866553e0fd22139477bf1
Parents: 8596ccf
Author: Hongbin Ma 
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 16:52:19 2016 +0800

--
 .../org/apache/kylin/common/util/CheckUtil.java | 33 
 .../apache/kylin/rest/service/QueryService.java | 17 ++
 2 files changed, 44 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message, 
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.rest.service;
 
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
 import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
 long startTime = System.currentTimeMillis();
 
 SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && 
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled = 
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in 
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query 
cache disabled in BackdoorToggles");
+
 if (queryCacheEnabled) {
 sqlResponse = searchQueryInCache(sqlRequest);
 }
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
 sqlResponse.setDuration(System.currentTimeMillis() - 
startTime);
 logger.info("Stats of SQL response: isException: {}, 
duration: {}, total scan count {}", //
 String.valueOf(sqlResponse.getIsException()), 
String.valueOf(sqlResponse.getDuration()), 
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold || 
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() < 
kylinConfig.getLargeQueryThreshold())) { //don't cache too large 

[8/8] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[lidong]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7d00933
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7d00933
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7d00933

Branch: refs/heads/yang22-cdh5.7
Commit: d7d009339ce4d916ec93b005573e22a5c2c0aef1
Parents: 49b8d18
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Sun Dec 25 20:15:17 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d7d00933/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/d7d00933/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d7d00933/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[4/7] kylin git commit: close OLAPEnumerator at exception

[lidong]

close OLAPEnumerator at exception


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f9dc7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f9dc7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f9dc7a

Branch: refs/heads/yang22-hbase1.x
Commit: 83f9dc7a68e64c8c71dd3dea971f9f0311a5575c
Parents: 6bb1382
Author: Hongbin Ma 
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 18:19:02 2016 +0800

--
 .../kylin/query/enumerator/OLAPEnumerator.java  | 32 +---
 1 file changed, 21 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/83f9dc7a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..408d227 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,30 @@ public class OLAPEnumerator implements Enumerator 
{
 
 @Override
 public boolean moveNext() {
-if (cursor == null) {
-cursor = queryStorage();
-}
+try {
+if (cursor == null) {
+cursor = queryStorage();
+}
 
-if (!cursor.hasNext()) {
-return false;
-}
+if (!cursor.hasNext()) {
+return false;
+}
 
-ITuple tuple = cursor.next();
-if (tuple == null) {
-return false;
+ITuple tuple = cursor.next();
+if (tuple == null) {
+return false;
+}
+convertCurrentRow(tuple);
+return true;
+} finally {
+try {
+if (cursor != null) {
+cursor.close();
+}
+} catch (Exception e) {
+logger.info("Error when closing cursor, ignore it", e);
+}
 }
-convertCurrentRow(tuple);
-return true;
 }
 
 private Object[] convertCurrentRow(ITuple tuple) {

[6/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
 System.out.println();
 
 descriptor.setValue(IRealizationConstants.HTableOwner, 
"dl-ebay-ky...@ebay.com");
-hbaseAdmin.modifyTable(descriptor.getNameAsString(), 
descriptor);
+
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), 
descriptor);
 }
 }
 hbaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.JsonSerializer;
 import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
 private static ResourceStore srcStore;
 private static ResourceStore dstStore;
 private static FileSystem hdfsFS;
-private static HBaseAdmin hbaseAdmin;
+private static Admin hbaseAdmin;
 
 public static final String ACL_INFO_FAMILY = "i";
 private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
 
 checkAndGetHbaseUrl();
 
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+hbaseAdmin = conn.getAdmin();
 
 hdfsFS = FileSystem.get(new Configuration());
 
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
 operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] 
{ item, cube.getName() }));
 }
 }
+
 private static void addCubeAndModelIntoProject(CubeInstance srcCube, 
String cubeName, String projectName) throws IOException {
 String projectResPath = 
ProjectInstance.concatResourcePath(projectName);
 if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class 

[3/7] kylin git commit: strenthen storage visit deadline calculation

[lidong]

strenthen storage visit deadline calculation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6bb13829
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6bb13829
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6bb13829

Branch: refs/heads/yang22-hbase1.x
Commit: 6bb1382970f8581b3f8a666ac98829e2a501ffd6
Parents: 6e18274
Author: Hongbin Ma 
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 17:57:55 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6bb13829/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 }
 
 final MutableBoolean scanNormalComplete = new MutableBoolean(true);
-final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+final long deadline = scanReq.getStartTime() + 
scanReq.getTimeout();
 logger.info("deadline is " + deadline);
 final long storagePushDownLimit = 
scanReq.getStoragePushDownLimit();
 

[7/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[lidong]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49b8d186
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49b8d186
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49b8d186

Branch: refs/heads/yang22-hbase1.x
Commit: 49b8d1865b954a4395fe5965988be9297a553454
Parents: 473f77b
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Sun Dec 25 20:13:27 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 417 insertions(+), 472 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/49b8d186/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[2/7] kylin git commit: introduce CheckUtil to make multiple condition checks clearer in log

[lidong]

introduce CheckUtil to make multiple condition checks clearer in log


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e18274c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e18274c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e18274c

Branch: refs/heads/yang22-hbase1.x
Commit: 6e18274c150e21fc952866553e0fd22139477bf1
Parents: 8596ccf
Author: Hongbin Ma 
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Dec 24 16:52:19 2016 +0800

--
 .../org/apache/kylin/common/util/CheckUtil.java | 33 
 .../apache/kylin/rest/service/QueryService.java | 17 ++
 2 files changed, 44 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+public static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(CheckUtil.class);
+
+public static boolean checkCondition(boolean condition, String message, 
Object... args) {
+if (condition) {
+return true;
+} else {
+logger.debug(message, args);
+return false;
+}
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e18274c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.rest.service;
 
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
 import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
 long startTime = System.currentTimeMillis();
 
 SQLResponse sqlResponse = null;
-boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && 
!BackdoorToggles.getDisableCache();
+boolean queryCacheEnabled = 
checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in 
KylinConfig") && //
+checkCondition(!BackdoorToggles.getDisableCache(), "query 
cache disabled in BackdoorToggles");
+
 if (queryCacheEnabled) {
 sqlResponse = searchQueryInCache(sqlRequest);
 }
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
 sqlResponse.setDuration(System.currentTimeMillis() - 
startTime);
 logger.info("Stats of SQL response: isException: {}, 
duration: {}, total scan count {}", //
 String.valueOf(sqlResponse.getIsException()), 
String.valueOf(sqlResponse.getDuration()), 
String.valueOf(sqlResponse.getTotalScanCount()));
-if (queryCacheEnabled && //
-!sqlResponse.getIsException() && //
-(sqlResponse.getDuration() > durationThreshold || 
sqlResponse.getTotalScanCount() > scancountThreshold) && //
-(sqlResponse.getResults().size() < 
kylinConfig.getLargeQueryThreshold())) { //don't cache too large 

[5/7] kylin git commit: bug fix

[lidong]

bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/473f77b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/473f77b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/473f77b0

Branch: refs/heads/yang22-hbase1.x
Commit: 473f77b06a1554aedbfcfd4fc5b827087aadb426
Parents: 83f9dc7
Author: Hongbin Ma 
Authored: Sun Dec 25 14:51:08 2016 +0800
Committer: Hongbin Ma 
Committed: Sun Dec 25 14:51:08 2016 +0800

--
 .../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/473f77b0/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java 
b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 408d227..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -76,14 +76,15 @@ public class OLAPEnumerator implements Enumerator 
{
 }
 convertCurrentRow(tuple);
 return true;
-} finally {
+} catch (Exception e) {
 try {
 if (cursor != null) {
 cursor.close();
 }
-} catch (Exception e) {
-logger.info("Error when closing cursor, ignore it", e);
+} catch (Exception ee) {
+logger.info("Error when closing cursor, ignore it", ee);
 }
+throw e;
 }
 }
 

[1/3] kylin git commit: spark cubing init commit

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/sparkcubing [created] 2e0665c9d


http://git-wip-us.apache.org/repos/asf/kylin/blob/2e0665c9/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..763b537 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.collect.Sets;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -56,8 +57,6 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Sets;
-
 public class HiveMRInput implements IMRInput {
 
 public static String getTableNameForHCat(TableDesc table) {
@@ -74,7 +73,7 @@ public class HiveMRInput implements IMRInput {
 public IMRTableInputFormat getTableInputFormat(TableDesc table) {
 return new HiveTableInputFormat(getTableNameForHCat(table));
 }
-
+
 @Override
 public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) {
 return new IMRBatchMergeInputSide() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2e0665c9/tool/pom.xml
--
diff --git a/tool/pom.xml b/tool/pom.xml
index 8eddec4..b577ae9 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -40,6 +40,10 @@
 
 
 org.apache.kylin
+kylin-engine-spark
+
+
+org.apache.kylin
 kylin-source-kafka
 
 

[3/3] kylin git commit: spark cubing init commit

[shaofengshi]

spark cubing init commit


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e0665c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e0665c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e0665c9

Branch: refs/heads/sparkcubing
Commit: 2e0665c9d2ed5b24d5cafd3ff8f7334f4f3d5a49
Parents: d11d019
Author: shaofengshi 
Authored: Sun Dec 25 15:59:16 2016 +0800
Committer: shaofengshi 
Committed: Sun Dec 25 15:59:16 2016 +0800

--
 assembly/pom.xml|   5 +-
 .../main/config/assemblies/source-assembly.xml  |   3 -
 .../java/org/apache/kylin/job/DeployUtil.java   |   1 -
 build/conf/kylin-spark-conf.properties  |  27 ++
 .../apache/kylin/common/KylinConfigBase.java|  38 +-
 .../kylin/common/persistence/ResourceStore.java |  35 ++
 .../org/apache/kylin/common/util/Array.java |   2 +-
 .../apache/kylin/common/util/SplittedBytes.java |   2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   2 +-
 .../kylin/cube/common/RowKeySplitter.java   |  12 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java|  27 +-
 .../kylin/cube/cuboid/CuboidScheduler.java  |  15 +-
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|   6 +-
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |   7 +-
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|   2 +-
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java |  19 +-
 .../kylin/cube/model/AggregationGroup.java  |   4 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |   2 +-
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   6 +-
 .../apache/kylin/cube/model/DictionaryDesc.java |   2 +-
 .../apache/kylin/cube/model/DimensionDesc.java  |   2 +-
 .../kylin/cube/model/HBaseColumnDesc.java   |  11 +-
 .../kylin/cube/model/HBaseColumnFamilyDesc.java |   9 +-
 .../kylin/cube/model/HBaseMappingDesc.java  |  15 +-
 .../apache/kylin/cube/model/HierarchyDesc.java  |   2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |   2 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java |  17 +-
 .../org/apache/kylin/cube/model/SelectRule.java |   2 +-
 .../dict/NumberDictionaryForestBuilder.java |   2 +-
 .../apache/kylin/dict/StringBytesConverter.java |   2 +-
 .../kylin/job/execution/ExecutableManager.java  |  21 +-
 .../kylin/dimension/AbstractDateDimEnc.java |  12 +-
 .../apache/kylin/dimension/BooleanDimEnc.java   |   4 +-
 .../kylin/dimension/DictionaryDimEnc.java   |  21 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  |   4 +-
 .../kylin/dimension/FixedLenHexDimEnc.java  |   4 +-
 .../org/apache/kylin/dimension/IntDimEnc.java   |   4 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   4 +-
 .../kylin/dimension/OneMoreByteVLongDimEnc.java |  14 +-
 .../kylin/measure/BufferedMeasureCodec.java |  10 +-
 .../org/apache/kylin/measure/MeasureCodec.java  |   8 +-
 .../apache/kylin/measure/MeasureIngester.java   |  12 +-
 .../org/apache/kylin/measure/MeasureType.java   |  12 +-
 .../kylin/measure/basic/DoubleIngester.java |   5 +
 .../measure/basic/DoubleSumAggregator.java  |   3 +-
 .../kylin/measure/basic/LongIngester.java   |   5 +
 .../kylin/measure/basic/LongSumAggregator.java  |   3 +-
 .../kylin/measure/bitmap/BitmapCounter.java |   8 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |   5 +
 .../kylin/measure/hllc/HLLCMeasureType.java |   5 +
 .../kylin/measure/hllc/HLLCSerializer.java  |  11 +-
 .../apache/kylin/measure/raw/RawSerializer.java |   4 +-
 .../apache/kylin/measure/topn/TopNCounter.java  |   2 +-
 .../kylin/measure/topn/TopNMeasureType.java |   4 +-
 .../metadata/datatype/BigDecimalSerializer.java |   2 +-
 .../metadata/datatype/BooleanSerializer.java|   9 +-
 .../metadata/datatype/DataTypeSerializer.java   |  17 +-
 .../metadata/datatype/DateTimeSerializer.java   |   9 +-
 .../metadata/datatype/DoubleSerializer.java |   5 +-
 .../kylin/metadata/datatype/Int4Serializer.java |   9 +-
 .../metadata/datatype/Long8Serializer.java  |   9 +-
 .../kylin/metadata/datatype/LongSerializer.java |   9 +-
 .../kylin/metadata/model/FunctionDesc.java  |  22 +-
 .../apache/kylin/metadata/model/JoinDesc.java   |   7 +-
 .../kylin/metadata/model/JoinTableDesc.java |   4 +-
 .../apache/kylin/metadata/model/JoinsTree.java  |   9 +-
 .../kylin/metadata/model/MeasureDesc.java   |   9 +-
 .../metadata/model/ModelDimensionDesc.java  |  10 +-
 .../kylin/metadata/model/ParameterDesc.java |  13 +-
 .../kylin/metadata/model/PartitionDesc.java |   6 +-
 .../apache/kylin/metadata/model/TableRef.java   |  17 +-
 .../kylin/engine/mr/BatchCubingJobBuilder2.java |   9 +-
 .../kylin/engine/mr/JobBuilderSupport.java  |  12 +-
 .../engine/mr/common/BaseCuboidBuilder.java | 175 +