from:"mahongbin"

[8/8] kylin git commit: KYLIN-1858 remove all ii related code

2016-07-08 Thread mahongbin

KYLIN-1858 remove all ii related code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2cc0b9c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2cc0b9c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2cc0b9c4

Branch: refs/heads/master
Commit: 2cc0b9c43f77624dfaa09b6410a818ac020fb475
Parents: cd5e603
Author: Hongbin Ma 
Authored: Fri Jul 8 15:14:37 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Jul 8 15:14:37 2016 +0800

--
 assembly/pom.xml|6 -
 .../kylin/job/hadoop/invertedindex/IICLI.java   |  106 -
 .../kylin/job/hadoop/invertedindex/IITest.java  |  266 --
 .../apache/kylin/common/KylinConfigBase.java|2 +-
 .../kylin/common/persistence/ResourceStore.java |2 -
 .../org/apache/kylin/common/util/ClassUtil.java |1 -
 .../common/util/AbstractKylinTestCase.java  |2 -
 engine-mr/pom.xml   |5 -
 .../org/apache/kylin/engine/mr/IMROutput.java   |   26 +-
 .../kylin/engine/mr/JobBuilderSupport.java  |9 -
 .../java/org/apache/kylin/engine/mr/MRUtil.java |5 -
 .../engine/mr/common/AbstractHadoopJob.java |   18 -
 .../mr/invertedindex/BatchIIJobBuilder.java |   84 -
 .../kylin/engine/mr/invertedindex/IIJob.java|   71 -
 .../mr/invertedindex/InvertedIndexJob.java  |  135 -
 .../mr/invertedindex/InvertedIndexMapper.java   |   77 -
 .../invertedindex/InvertedIndexPartitioner.java |   73 -
 .../mr/invertedindex/InvertedIndexReducer.java  |  128 -
 .../UpdateIIInfoAfterBuildStep.java |   83 -
 engine-streaming/pom.xml|5 -
 .../invertedindex/test_kylin_ii_inner_join.json |   16 -
 .../invertedindex/test_kylin_ii_left_join.json  |   15 -
 .../invertedindex/test_streaming_table_ii.json  |   15 -
 .../test_kylin_ii_inner_join_desc.json  |   69 -
 .../test_kylin_ii_left_join_desc.json   |   69 -
 .../test_streaming_table_ii_desc.json   |   26 -
 .../localmeta/project/default.json  |   10 -
 .../.settings/org.eclipse.core.resources.prefs  |6 -
 .../.settings/org.eclipse.jdt.core.prefs|  386 --
 .../.settings/org.eclipse.jdt.ui.prefs  |7 -
 invertedindex/pom.xml   |  103 -
 .../invertedindex/IICapabilityChecker.java  |  182 -
 .../kylin/invertedindex/IIDescManager.java  |  251 --
 .../apache/kylin/invertedindex/IIInstance.java  |  369 --
 .../apache/kylin/invertedindex/IIManager.java   |  272 --
 .../apache/kylin/invertedindex/IISegment.java   |  279 --
 .../invertedindex/index/BatchSliceMaker.java|   95 -
 .../invertedindex/index/BitMapContainer.java|  227 --
 .../index/ColumnValueContainer.java |   43 -
 .../index/CompressedValueContainer.java |  184 -
 .../index/IncrementalSliceMaker.java|  123 -
 .../invertedindex/index/RawTableRecord.java |  133 -
 .../index/ShardingSliceBuilder.java |   53 -
 .../apache/kylin/invertedindex/index/Slice.java |  215 -
 .../kylin/invertedindex/index/SliceBuilder.java |   78 -
 .../kylin/invertedindex/index/TableRecord.java  |  193 -
 .../invertedindex/index/TableRecordInfo.java|  147 -
 .../index/TableRecordInfoDigest.java|  175 -
 .../invertedindex/measure/FixedHLLCodec.java|   78 -
 .../measure/FixedLenMeasureCodec.java   |   50 -
 .../measure/FixedPointLongCodec.java|  117 -
 .../kylin/invertedindex/model/IIDesc.java   |  418 --
 .../model/IIJoinedFlatTableDesc.java|   68 -
 .../invertedindex/model/IIKeyValueCodec.java|  235 --
 .../model/IIKeyValueCodecWithState.java |  100 -
 .../apache/kylin/invertedindex/model/IIRow.java |   90 -
 .../invertedindex/model/KeyValueCodec.java  |   48 -
 .../invertedindex/util/IIDictionaryBuilder.java |   86 -
 .../kylin/invertedindex/IIDescManagerTest.java  |  103 -
 .../apache/kylin/invertedindex/IIDescTest.java  |   66 -
 .../kylin/invertedindex/IIInstanceTest.java |   70 -
 .../invertedindex/InvertedIndexLocalTest.java   |  264 --
 .../org/apache/kylin/invertedindex/LZFTest.java |   49 -
 .../measure/FixedPointLongCodecTest.java|   62 -
 kylin-it/pom.xml|6 -
 .../apache/kylin/query/ITCombinationTest.java   |1 -
 .../org/apache/kylin/query/ITIIQueryTest.java   |   92 -
 .../apache/kylin/query/ITKylinQueryTest.java|1 -
 .../hbase/ii/ITInvertedIndexHBaseTest.java  |  115 -
 pom.xml |1 -
 query/pom.xml   |6 -
 .../apache/kylin/query/routing/Candidate.java   |1 -
 .../apache/kylin/rest/service/BasicService.java |   10 -
 .../apache/kylin/rest/service/CacheService.java |   22 -
 .../apache/kylin/rest/service/ModelService.java |9 -

[5/8] kylin git commit: KYLIN-1858 remove all ii related code

2016-07-08 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
--
diff --git 
a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
 
b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
deleted file mode 100644
index dfaf106..000
--- 
a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionarySerializer;
-import org.apache.kylin.invertedindex.index.ColumnValueContainer;
-import org.apache.kylin.invertedindex.index.CompressedValueContainer;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.datatype.DataType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9
- */
-public class IIKeyValueCodec implements KeyValueCodec {
-
-public static final int SHARD_LEN = 2;
-public static final int TIMEPART_LEN = 8;
-public static final int COLNO_LEN = 2;
-protected final TableRecordInfoDigest digest;
-
-public IIKeyValueCodec(TableRecordInfoDigest digest) {
-this.digest = digest;
-}
-
-@Override
-public Collection encodeKeyValue(Slice slice) {
-ArrayList result = Lists.newArrayList();
-ColumnValueContainer[] containers = slice.getColumnValueContainers();
-for (int col = 0; col < containers.length; col++) {
-if (containers[col] instanceof CompressedValueContainer) {
-final IIRow row = collectKeyValues(slice, col, 
(CompressedValueContainer) containers[col]);
-result.add(row);
-} else {
-throw new IllegalArgumentException("Unknown container class " 
+ containers[col].getClass());
-}
-}
-return result;
-}
-
-private IIRow collectKeyValues(Slice slice, int col, 
CompressedValueContainer container) {
-ImmutableBytesWritable key = encodeKey(slice.getShard(), 
slice.getTimestamp(), col);
-ImmutableBytesWritable value = container.toBytes();
-final Dictionary dictionary = slice.getLocalDictionaries() != null 
? slice.getLocalDictionaries()[col] : null;
-if (dictionary == null) {
-return new IIRow(key, value, new 
ImmutableBytesWritable(BytesUtil.EMPTY_BYTE_ARRAY));
-} else {
-ByteArray bytes = DictionarySerializer.serialize(dictionary);
-return new IIRow(key, value, new 
ImmutableBytesWritable(bytes.array(), bytes.offset(), bytes.length()));
-}
-}
-
-ImmutableBytesWritable encodeKey(short shard, long timestamp, int col) {
-byte[] bytes = new byte[20];
-int len = encodeKey(shard, timestamp, col, bytes, 0);
-return new ImmutableBytesWritable(bytes, 0, len);
-}
-
-int encodeKey(short shard, long timestamp, int col, byte[] buf, int 
offset) {
-int i = offset;
-
-BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN);
-i += SHARD_LEN;
-BytesUtil.writeLong(timestamp, buf, i, TIMEPART_LEN);
-i += TIMEPART_LEN;
-
-BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN);
-i += COLNO_LEN;
-
-return i - offset;
-}
-
-@Override
-public Iterable decodeKeyValue(Iterable kvs) {
-return new IIRowDecoder(digest, kvs.iterator());
-//return new Decoder(kvs, incompleteDigest);
-}
-
-private static TableRecordInfoDigest

[4/8] kylin git commit: KYLIN-1858 remove all ii related code

2016-07-08 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
deleted file mode 100644
index 545d058..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.ii;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.engine.mr.KylinMapper;
-import org.apache.kylin.invertedindex.model.IIDesc;
-
-/**
- * @author yangli9
- */
-public class IICreateHFileMapper extends KylinMapper {
-
-long timestamp;
-
-@Override
-protected void setup(Context context) throws IOException, 
InterruptedException {
-super.bindCurrentConfiguration(context.getConfiguration());
-
-timestamp = System.currentTimeMillis();
-}
-
-@Override
-protected void map(ImmutableBytesWritable key, ImmutableBytesWritable 
value, Context context) throws IOException, InterruptedException {
-
-ByteBuffer buffer = ByteBuffer.wrap(value.get(), value.getOffset(), 
value.getLength());
-int totalLength = value.getLength();
-int valueLength = buffer.getInt();
-int dictionaryLength = totalLength - valueLength - 4;
-KeyValue kv = new KeyValue(key.get(), key.getOffset(), 
key.getLength(), //
-IIDesc.HBASE_FAMILY_BYTES, 0, 
IIDesc.HBASE_FAMILY_BYTES.length, //
-IIDesc.HBASE_QUALIFIER_BYTES, 0, 
IIDesc.HBASE_QUALIFIER_BYTES.length, //
-timestamp, Type.Put, //
-buffer.array(), buffer.position(), valueLength);
-
-// write value
-context.write(key, kv);
-
-kv = new KeyValue(key.get(), key.getOffset(), key.getLength(), //
-IIDesc.HBASE_FAMILY_BYTES, 0, 
IIDesc.HBASE_FAMILY_BYTES.length, //
-IIDesc.HBASE_DICTIONARY_BYTES, 0, 
IIDesc.HBASE_DICTIONARY_BYTES.length, //
-timestamp, Type.Put, //
-buffer.array(), buffer.position() + valueLength, 
dictionaryLength);
-
-// write dictionary
-context.write(key, kv);
-
-}
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
deleted file mode 100644
index 8099276..000
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHTableJob.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.ii;
-
-import org.apache.commons.cli.Options;
-import

kylin git commit: minor, renaming the left join cube with view

2016-07-07 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master db5f89bae -> cd5e603dd


minor, renaming the left join cube with view


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd5e603d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd5e603d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd5e603d

Branch: refs/heads/master
Commit: cd5e603dd55a785a3c23e92b1c24c3e8200f342f
Parents: db5f89b
Author: Hongbin Ma 
Authored: Thu Jul 7 17:37:02 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jul 7 17:37:02 2016 +0800

--
 .../cube/test_kylin_cube_with_view_empty.json   |  10 --
 ...st_kylin_cube_with_view_left_join_empty.json |  10 ++
 .../test_kylin_cube_with_view_desc.json | 169 ---
 ...est_kylin_cube_with_view_left_join_desc.json | 169 +++
 .../localmeta/project/default.json  |   4 +-
 .../kylin/provision/BuildCubeWithEngine.java|   2 +-
 6 files changed, 182 insertions(+), 182 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd5e603d/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_empty.json
--
diff --git 
a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_empty.json 
b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_empty.json
deleted file mode 100644
index ed42df1..000
--- 
a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_empty.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-  "uuid" : "3e80-41be-49a5-90ca-9fb7294db186",
-  "last_modified" : 0,
-  "name" : "test_kylin_cube_with_view_empty",
-  "owner" : null,
-  "descriptor" : "test_kylin_cube_with_view_desc",
-  "segments" : [ ],
-  "status" : "DISABLED",
-  "create_time" : null
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd5e603d/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json
--
diff --git 
a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json
 
b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json
new file mode 100644
index 000..62b7432
--- /dev/null
+++ 
b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json
@@ -0,0 +1,10 @@
+{
+  "uuid" : "3e80-41be-49a5-90ca-9fb7294db186",
+  "last_modified" : 0,
+  "name" : "test_kylin_cube_with_view_left_join_empty",
+  "owner" : null,
+  "descriptor" : "test_kylin_cube_with_view_left_join_desc",
+  "segments" : [ ],
+  "status" : "DISABLED",
+  "create_time" : null
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd5e603d/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json
deleted file mode 100644
index e66dc24..000
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_desc.json
+++ /dev/null
@@ -1,169 +0,0 @@
-{
-  "uuid" : "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
-  "name" : "test_kylin_cube_with_view_desc",
-  "description" : null,
-  "dimensions" : [ {
-"name" : "CAL_DT",
-"table" : "EDW.V_TEST_CAL_DT",
-"column" : "{FK}",
-"derived" : [ "WEEK_BEG_DT" ]
-  }, {
-"name" : "CATEGORY",
-"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
-"column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER" ]
-  }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
-"column" : "META_CATEG_NAME",
-"derived" : null
-  }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
-"column" : "CATEG_LVL2_NAME",
-"derived" : null
-  }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
-"column" : "CATEG_LVL3_NAME",
-"derived" : null
-  }, {
-"name" : "LSTG_FORMAT_NAME",
-"table" : "DEFAULT.TEST_KYLIN_FACT",
-"column" : "LSTG_FORMAT_NAME",
-"derived" : null
-  }, {
-"name" : "SITE_ID",
-"table" : "EDW.TEST_SITES",
-"column" : "{FK}",
-"derived" : [ "SITE_NAME", "CRE_USER" ]
-  }, {
-"name" : "SELLER_TYPE_CD",
-"table" : "EDW.TEST_SELLER_TYPE_DIM",
-"column" : "{FK}",
-"derived" : [ "SELLER_TYPE_DESC" ]
-  } ],
-  "measures" : [ {
-"name" : "GMV_SUM",
-"function" : {
-  "expression" : "SUM",
-  "parameter" : {
-"type" : "column",
-"value" : "PRICE",
-

[2/2] kylin git commit: minor, remove build ii step

2016-07-07 Thread mahongbin

minor, remove build ii step


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db5f89ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db5f89ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db5f89ba

Branch: refs/heads/master
Commit: db5f89bae53c9fa84c96c5c4693a142a136ef1fe
Parents: edfb37d
Author: Hongbin Ma 
Authored: Thu Jul 7 17:06:40 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jul 7 17:06:40 2016 +0800

--
 kylin-it/pom.xml|  38 +--
 .../kylin/provision/BuildCubeWithStream.java|  14 +-
 .../kylin/provision/BuildIIWithEngine.java  | 253 
 .../kylin/provision/BuildIIWithStream.java  | 300 ---
 4 files changed, 23 insertions(+), 582 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db5f89ba/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index eeb74fe..271bae8 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -18,7 +18,8 @@
 -->
 
 
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
 kylin
 org.apache.kylin
@@ -30,8 +31,8 @@
 Kylin:IT
 
 
-
-
+
+
 
 
 
@@ -283,9 +284,13 @@
 
 
 integration-tests
-integration-test
 
 integration-test
+
+
+
+verify
+
 verify
 
 
@@ -323,7 +328,7 @@
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
 -classpath
-
+
 
org.apache.kylin.provision.BuildCubeWithEngine
 
 
${project.basedir}
@@ -344,33 +349,12 @@
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
 -classpath
-
+
 
org.apache.kylin.provision.BuildCubeWithStream
 
 
${project.basedir}
 
 
-
-build_ii_with_stream
-
-exec
-
-pre-integration-test
-
-${skipTests}
-test
-java
-
--DuseSandbox=true
-
-Dhdp.version=${hdp.version}
-
-DfastBuildMode=${fastBuildMode}
--classpath
-
-
org.apache.kylin.provision.BuildIIWithStream
-
-
${project.basedir}
-
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/db5f89ba/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index c426ea4..1655a17 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++

kylin git commit: Minor changes: add dimension length info in CubeStatsReader

2016-07-06 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master acf1bc331 -> 4cd733a14


Minor changes: add dimension length info in CubeStatsReader


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4cd733a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4cd733a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4cd733a1

Branch: refs/heads/master
Commit: 4cd733a1465a5641229954afa5d8a6349cc6bb8a
Parents: acf1bc3
Author: Hongbin Ma 
Authored: Wed Jul 6 18:03:12 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jul 6 18:03:19 2016 +0800

--
 .../apache/kylin/engine/mr/common/CubeStatsReader.java| 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4cd733a1/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index cab0c8d..83e46e3 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -49,6 +49,7 @@ import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.cube.kv.CubeDimEncMap;
+import org.apache.kylin.cube.kv.RowKeyEncoder;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
@@ -233,6 +234,7 @@ public class CubeStatsReader {
 out.println("Total estimated size(MB): " + 
SumHelper.sumDouble(cuboidSizes.values()));
 out.println("Sampling percentage:  " + samplingPercentage);
 out.println("Mapper overlap ratio: " + mapperOverlapRatioOfFirstBuild);
+printKVInfo(out);
 printCuboidInfoTreeEntry(cuboidRows, cuboidSizes, out);
 
out.println("");
 }
@@ -245,6 +247,14 @@ public class CubeStatsReader {
 printCuboidInfoTree(-1L, baseCuboid, scheduler, cuboidRows, 
cuboidSizes, dimensionCount, 0, out);
 }
 
+private void printKVInfo(PrintWriter writer) {
+Cuboid cuboid = Cuboid.getBaseCuboid(seg.getCubeDesc());
+RowKeyEncoder encoder = new RowKeyEncoder(seg, cuboid);
+for (TblColRef col : cuboid.getColumns()) {
+writer.println("Length of dimension " + col + " is " + 
encoder.getColumnLength(col));
+}
+}
+
 private static void printCuboidInfoTree(long parent, long cuboidID, final 
CuboidScheduler scheduler, Map cuboidRows, Map 
cuboidSizes, int dimensionCount, int depth, PrintWriter out) {
 printOneCuboidInfo(parent, cuboidID, cuboidRows, cuboidSizes, 
dimensionCount, depth, out);

kylin git commit: add helper methods to gt scan request

2016-06-30 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 832112215 -> 07511d7bc


add helper methods to gt scan request


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07511d7b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07511d7b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07511d7b

Branch: refs/heads/master
Commit: 07511d7bc665008db85c45a003486c01f3d9c8da
Parents: 8321122
Author: Hongbin Ma 
Authored: Wed Jun 29 18:37:42 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jun 30 15:11:09 2016 +0800

--
 .../src/main/java/org/apache/kylin/gridtable/GTInfo.java | 10 ++
 .../java/org/apache/kylin/gridtable/GTScanRequest.java   | 11 ++-
 2 files changed, 20 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07511d7b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index 0571c6f..673d22e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -271,6 +271,14 @@ public class GTInfo {
 public IGTCodeSystem getCodeSystem() {
 return codeSystem;
 }
+
+public int getMaxLength() {
+int ret = 0;
+for (int i = 0; i < colAll.trueBitCount(); i++) {
+ret += codeSystem.maxCodeLength(colAll.trueBitAt(i));
+}
+return ret;
+}
 
 public static final BytesSerializer serializer = new 
BytesSerializer() {
 @Override
@@ -340,4 +348,6 @@ public class GTInfo {
 enableRowBlock(newRowBlockSize).build();
 }
 };
+
+  
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/07511d7b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 5e1e131..d4e6298 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -267,6 +267,16 @@ public class GTScanRequest {
 aggrCacheGB = -limit;
 }
 
+public List getRequiredMeasures() {
+List measures = Lists.newArrayList();
+int numDim = info.getPrimaryKey().trueBitCount();
+for (int i = 0; i < aggrMetrics.trueBitCount(); i++) {
+int index = aggrMetrics.trueBitAt(i);
+measures.add(index - numDim);
+}
+return measures;
+}
+
 @Override
 public String toString() {
 return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", 
filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", 
aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + 
Arrays.toString(aggrMetricsFuncs) + "]";
@@ -354,5 +364,4 @@ public class GTScanRequest {
 }
 
 };
-
 }

[02/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/src/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/src/.settings/org.eclipse.jdt.core.prefs 
b/src/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 000..5aaaf1e
--- /dev/null
+++ b/src/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,386 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
+org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
+org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
+org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
+org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
+org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
+org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
+org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
+org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
+org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
+org.eclipse.jdt.core.compiler.problem.deadCode=warning
+org.eclipse.jdt.core.compiler.problem.deprecation=warning
+org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
+org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
+org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
+org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
+org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
+org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
+org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
+org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
+org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
+org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
+org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
+org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
+org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
+org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
+org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
+org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
+org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
+org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
+org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
+org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
+org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
+org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
+org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
+org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
+org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
+org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
+org.eclipse.jdt.core.compiler.problem.nullReference=warning
+org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
+org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
+org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
+org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
+org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore

[08/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/engine-streaming/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/engine-streaming/.settings/org.eclipse.jdt.core.prefs 
b/engine-streaming/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/engine-streaming/.settings/org.eclipse.jdt.core.prefs
+++ b/engine-streaming/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore

[13/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/core-common/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/core-common/.settings/org.eclipse.jdt.core.prefs 
b/core-common/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/core-common/.settings/org.eclipse.jdt.core.prefs
+++ b/core-common/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore

[14/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

KYLIN-1821 keep all .settings same


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b46ed14
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b46ed14
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b46ed14

Branch: refs/heads/master
Commit: 7b46ed14cee13c2c47a7158e413ba742e9f4660c
Parents: 1044f1d
Author: Hongbin Ma 
Authored: Mon Jun 27 09:54:48 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Jun 27 09:55:52 2016 +0800

--
 .settings/org.eclipse.core.resources.prefs  |   2 -
 .../.settings/org.eclipse.core.resources.prefs  |  11 +-
 assembly/.settings/org.eclipse.jdt.core.prefs   | 772 +--
 assembly/.settings/org.eclipse.jdt.ui.prefs |  14 +-
 atopcalcite/.settings/org.eclipse.jdt.ui.prefs  |   2 +-
 .../.settings/org.eclipse.core.resources.prefs  |  11 +-
 .../.settings/org.eclipse.jdt.core.prefs| 772 +--
 core-common/.settings/org.eclipse.jdt.ui.prefs  |  14 +-
 .../.settings/org.eclipse.core.resources.prefs  |  11 +-
 core-cube/.settings/org.eclipse.jdt.core.prefs  | 772 +--
 core-cube/.settings/org.eclipse.jdt.ui.prefs|  14 +-
 .../.settings/org.eclipse.core.resources.prefs  |  11 +-
 .../.settings/org.eclipse.jdt.core.prefs| 772 +--
 .../.settings/org.eclipse.jdt.ui.prefs  |  14 +-
 core-job/.settings/org.eclipse.jdt.ui.prefs |   2 +-
 .../.settings/org.eclipse.core.resources.prefs  |  12 +-
 .../.settings/org.eclipse.jdt.core.prefs| 772 +--
 .../.settings/org.eclipse.jdt.ui.prefs  |  14 +-
 core-storage/.settings/org.eclipse.jdt.ui.prefs |   2 +-
 engine-mr/.settings/org.eclipse.jdt.ui.prefs|   2 +-
 .../.settings/org.eclipse.core.resources.prefs  |  11 +-
 .../.settings/org.eclipse.jdt.core.prefs| 772 +--
 engine-spark/.settings/org.eclipse.jdt.ui.prefs |  14 +-
 .../.settings/org.eclipse.core.resources.prefs  |  11 +-
 .../.settings/org.eclipse.jdt.core.prefs| 772 +--
 .../.settings/org.eclipse.jdt.ui.prefs  |  14 +-
 .../.settings/org.eclipse.core.resources.prefs  |   1 +
 .../.settings/org.eclipse.jdt.ui.prefs  |   2 +-
 jdbc/.settings/org.eclipse.core.resources.prefs |  11 +-
 jdbc/.settings/org.eclipse.jdt.core.prefs   | 772 +--
 jdbc/.settings/org.eclipse.jdt.ui.prefs |  14 +-
 .../.settings/org.eclipse.core.resources.prefs  |  12 +-
 kylin-it/.settings/org.eclipse.jdt.core.prefs   | 772 +--
 kylin-it/.settings/org.eclipse.jdt.ui.prefs |  14 +-
 .../.settings/org.eclipse.core.resources.prefs  |  12 +-
 query/.settings/org.eclipse.jdt.core.prefs  | 772 +--
 query/.settings/org.eclipse.jdt.ui.prefs|  14 +-
 .../.settings/org.eclipse.core.resources.prefs  |   1 +
 server-base/.settings/org.eclipse.jdt.ui.prefs  |   2 +-
 .../.settings/org.eclipse.core.resources.prefs  |  11 +-
 server/.settings/org.eclipse.jdt.core.prefs | 772 +--
 server/.settings/org.eclipse.jdt.ui.prefs   |  14 +-
 source-hive/.settings/org.eclipse.jdt.ui.prefs  |   2 +-
 .../.settings/org.eclipse.core.resources.prefs  |  11 +-
 .../.settings/org.eclipse.jdt.core.prefs| 772 +--
 source-kafka/.settings/org.eclipse.jdt.ui.prefs |  14 +-
 src/.settings/org.eclipse.core.resources.prefs  |   6 +
 src/.settings/org.eclipse.jdt.core.prefs| 386 ++
 src/.settings/org.eclipse.jdt.ui.prefs  |   7 +
 .../.settings/org.eclipse.jdt.ui.prefs  |   2 +-
 tool/.settings/org.eclipse.core.resources.prefs |  11 +-
 tool/.settings/org.eclipse.jdt.core.prefs   | 772 +--
 tool/.settings/org.eclipse.jdt.ui.prefs |  14 +-
 53 files changed, 5596 insertions(+), 5187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/.settings/org.eclipse.core.resources.prefs
--
diff --git a/.settings/org.eclipse.core.resources.prefs 
b/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 99f26c0..000
--- a/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,2 +0,0 @@
-eclipse.preferences.version=1
-encoding/=UTF-8

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/assembly/.settings/org.eclipse.core.resources.prefs
--
diff --git a/assembly/.settings/org.eclipse.core.resources.prefs 
b/assembly/.settings/org.eclipse.core.resources.prefs
index 365bbd6..29abf99 100644
--- a/assembly/.settings/org.eclipse.core.resources.prefs
+++ b/assembly/.settings/org.eclipse.core.resources.prefs
@@ -1,5 +1,6 @@

[06/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/kylin-it/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/kylin-it/.settings/org.eclipse.jdt.core.prefs 
b/kylin-it/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/kylin-it/.settings/org.eclipse.jdt.core.prefs
+++ b/kylin-it/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore

[10/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/core-metadata/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/core-metadata/.settings/org.eclipse.jdt.core.prefs 
b/core-metadata/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/core-metadata/.settings/org.eclipse.jdt.core.prefs
+++ b/core-metadata/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore

[04/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/server/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/server/.settings/org.eclipse.jdt.core.prefs 
b/server/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/server/.settings/org.eclipse.jdt.core.prefs
+++ b/server/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore

[01/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 1044f1dbb -> 7b46ed14c


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/tool/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/tool/.settings/org.eclipse.jdt.core.prefs 
b/tool/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/tool/.settings/org.eclipse.jdt.core.prefs
+++ b/tool/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore

[12/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/core-cube/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/core-cube/.settings/org.eclipse.jdt.core.prefs 
b/core-cube/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/core-cube/.settings/org.eclipse.jdt.core.prefs
+++ b/core-cube/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore

[11/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/core-dictionary/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/core-dictionary/.settings/org.eclipse.jdt.core.prefs 
b/core-dictionary/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/core-dictionary/.settings/org.eclipse.jdt.core.prefs
+++ b/core-dictionary/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore

[09/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/engine-spark/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/engine-spark/.settings/org.eclipse.jdt.core.prefs 
b/engine-spark/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/engine-spark/.settings/org.eclipse.jdt.core.prefs
+++ b/engine-spark/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore

[05/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/query/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/query/.settings/org.eclipse.jdt.core.prefs 
b/query/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/query/.settings/org.eclipse.jdt.core.prefs
+++ b/query/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore

[03/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/source-kafka/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/source-kafka/.settings/org.eclipse.jdt.core.prefs 
b/source-kafka/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/source-kafka/.settings/org.eclipse.jdt.core.prefs
+++ b/source-kafka/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore

[07/14] kylin git commit: KYLIN-1821 keep all .settings same

2016-06-26 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7b46ed14/jdbc/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/jdbc/.settings/org.eclipse.jdt.core.prefs 
b/jdbc/.settings/org.eclipse.jdt.core.prefs
index 19e2439..5aaaf1e 100644
--- a/jdbc/.settings/org.eclipse.jdt.core.prefs
+++ b/jdbc/.settings/org.eclipse.jdt.core.prefs
@@ -1,386 +1,386 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
-org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
-org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
-org.eclipse.jdt.core.compiler.annotation.nonnull.secondary=
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
-org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
-org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
-org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
-org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
-org.eclipse.jdt.core.compiler.problem.deadCode=warning
-org.eclipse.jdt.core.compiler.problem.deprecation=warning
-org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
-org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
-org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore
-org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
-org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
-org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
-org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
-org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
-org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore
-org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
-org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
-org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
-org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
-org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
-org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
-org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
-org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
-org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
-org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
-org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
-org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
-org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
-org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
-org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
-org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
-org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
-org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning
-org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
-org.eclipse.jdt.core.compiler.problem.nullReference=warning
-org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
-org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
-org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
-org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning
-org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
-org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore

kylin git commit: KYLIN-1821 remaining

2016-06-25 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 81e3b2cd3 -> ef226f601


KYLIN-1821 remaining


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef226f60
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef226f60
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef226f60

Branch: refs/heads/master
Commit: ef226f60155cbbbc9783e59c9459d09ca6ecb167
Parents: 81e3b2c
Author: Hongbin Ma 
Authored: Sat Jun 25 21:19:29 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Jun 25 21:19:29 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/model/CubeDesc.java  | 4 ++--
 .../src/main/java/org/apache/kylin/rest/service/UserService.java | 2 +-
 .../org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java| 3 +--
 3 files changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef226f60/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 7d9c058..fbdeeb7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -710,13 +710,13 @@ public class CubeDesc extends RootPersistentEntity {
 
 if ((colStrs == null && dim.isDerived()) || 
("{FK}".equalsIgnoreCase(colStrs))) {
 // when column is omitted, special case
-
+
 for (TblColRef col : join.getForeignKeyColumns()) {
 dimCols.add(initDimensionColRef(col));
 }
 } else {
 // normal case
-
+
 if (StringUtils.isEmpty(colStrs))
 throw new IllegalStateException("Dimension column must not 
be blank " + dim);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef226f60/server/src/main/java/org/apache/kylin/rest/service/UserService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/UserService.java 
b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
index c1f4e98..07c7c6f 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -279,7 +279,7 @@ public class UserService implements UserDetailsManager {
 return false;
 return true;
 }
-
+
 @Override
 public String toString() {
 return authority;

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef226f60/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 3f67e5c..32aa4b0 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -541,8 +541,7 @@ public class CubeMigrationCLI {
 logger.info("Undo for PURGE_AND_DISABLE is not supported");
 break;
 }
-default:
-{
+default: {
 //do nothing
 break;
 }

kylin git commit: KYLIN-1821 checkstyle enable leftcurly and rightcurly rules

2016-06-25 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 5d4982e24 -> 81e3b2cd3


KYLIN-1821 checkstyle enable leftcurly and rightcurly rules


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81e3b2cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81e3b2cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81e3b2cd

Branch: refs/heads/master
Commit: 81e3b2cd3d0ee4632b95d80d54aafd355b541726
Parents: 5d4982e
Author: Hongbin Ma 
Authored: Sat Jun 25 19:47:48 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Jun 25 20:18:23 2016 +0800

--
 checkstyle.xml|  2 ++
 .../kylin/common/persistence/ResourceTool.java|  7 +++
 .../java/org/apache/kylin/cube/CubeManager.java   |  9 -
 .../kylin/cube/gridtable/CubeCodeSystem.java  |  7 +++
 .../org/apache/kylin/cube/model/CubeDesc.java |  9 +
 .../apache/kylin/cube/model/v1_4_0/CubeDesc.java  |  7 +++
 .../upgrade/common/MetadataVersionRefresher.java  |  9 +
 .../kylin/gridtable/GTAggregateScanner.java   |  9 +
 .../benchmark/SortedGTRecordGenerator.java|  9 +
 .../apache/kylin/dict/AppendTrieDictionary.java   | 18 --
 .../java/org/apache/kylin/dict/CachedTreeMap.java |  1 -
 .../apache/kylin/dict/TrieDictionaryBuilder.java  | 16 +++-
 .../measure/hllc/HyperLogLogPlusCounter.java  |  5 ++---
 .../storage/gtrecord/CubeTupleConverter.java  |  5 ++---
 .../kylin/query/enumerator/OLAPEnumerator.java|  7 +++
 .../storage/hbase/cube/v1/CubeTupleConverter.java |  7 +++
 .../endpoint/EndpointTupleConverter.java  | 12 +---
 .../apache/kylin/tool/util/ResourceStoreUtil.java |  7 +++
 18 files changed, 68 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/81e3b2cd/checkstyle.xml
--
diff --git a/checkstyle.xml b/checkstyle.xml
index 10ba67b..682186f 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -49,6 +49,8 @@
 
 
 
+
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/81e3b2cd/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index b10856f..2f16eb7 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -131,8 +131,8 @@ public class ResourceTool {
 public static void copyR(ResourceStore src, ResourceStore dst, String 
path) throws IOException {
 NavigableSet children = src.listResources(path);
 
-// case of resource (not a folder)
 if (children == null) {
+// case of resource (not a folder)
 if (matchFilter(path)) {
 try {
 RawResource res = src.getResource(path);
@@ -147,9 +147,8 @@ public class ResourceTool {
 ex.printStackTrace();
 }
 }
-}
-// case of folder
-else {
+} else {
+// case of folder
 for (String child : children)
 copyR(src, dst, child);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/81e3b2cd/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b62bc05..a200f5d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -766,18 +766,17 @@ public class CubeManager implements IRealizationProvider {
 continue;
 }
 
-// if i, j competes
 if (is.getSourceOffsetStart() == js.getSourceOffsetStart()) {
-// if both new or ready, favor the bigger segment
+// if i, j competes
 if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) {
+// if both new or ready, favor the bigger segment
 if (is.getSourceOffsetEnd() <= js.getSourceOffsetEnd()) {
 tobe.remove(i);
 } else {
 tobe.remove(j);
 }
-}
-

[16/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/checkstyle-suppressions.xml
--
diff --git a/checkstyle-suppressions.xml b/checkstyle-suppressions.xml
new file mode 100644
index 000..d247c65
--- /dev/null
+++ b/checkstyle-suppressions.xml
@@ -0,0 +1,37 @@
+
+http://www.puppycrawl.com/dtds/suppressions_1_0.dtd;>
+  
+
+  
+  
+  
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/checkstyle.xml
--
diff --git a/checkstyle.xml b/checkstyle.xml
new file mode 100644
index 000..10ba67b
--- /dev/null
+++ b/checkstyle.xml
@@ -0,0 +1,60 @@
+
+
+
+http://www.puppycrawl.com/dtds/configuration_1_3.dtd;>
+
+
+
+
+
+
+
+
+
+
+
+ftp://"/>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 8719541..283f505 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -371,22 +371,22 @@ public class KylinConfig extends KylinConfigBase {
 }
 
 private static void dumpStackTrace() {
-
+
 //uncomment below to start debugging
-
-//Thread t = Thread.currentThread();
-//int maxStackTraceDepth = 20;
-//int current = 0;
-//
-//StackTraceElement[] stackTrace = t.getStackTrace();
-//StringBuilder buf = new StringBuilder("This is not a exception, just 
for diagnose purpose:");
-//buf.append("\n");
-//for (StackTraceElement e : stackTrace) {
-//if (++current > maxStackTraceDepth) {
-//break;
-//}
-//buf.append("\t").append("at ").append(e.toString()).append("\n");
-//}
-//logger.info(buf.toString());
+
+//Thread t = Thread.currentThread();
+//int maxStackTraceDepth = 20;
+//int current = 0;
+//
+//StackTraceElement[] stackTrace = t.getStackTrace();
+//StringBuilder buf = new StringBuilder("This is not a 
exception, just for diagnose purpose:");
+//buf.append("\n");
+//for (StackTraceElement e : stackTrace) {
+//if (++current > maxStackTraceDepth) {
+//break;
+//}
+//buf.append("\t").append("at 
").append(e.toString()).append("\n");
+//}
+//logger.info(buf.toString());
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index c1c62fb..dcd3f38 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -74,7 +74,7 @@ public class FileResourceStore extends ResourceStore {
 NavigableSet resources = listResources(folderPath);
 if (resources == null)
 return Collections.emptyList();
-
+
 List result = 
Lists.newArrayListWithCapacity(resources.size());
 try {
 for (String res : resources) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
index d2a684e..2e7768a 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java
@@ -25,7 +25,7 @@ import java.io.IOException;
 import org.apache.kylin.common.util.JsonUtil;
 
 /**
- * @author yangli9
+ * @author yangli9  
  */
 public class JsonSerializer implements 
Serializer {

[12/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index bbd0fc5..44c80ee 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -21,7 +21,11 @@ package org.apache.kylin.engine.mr;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
 import java.util.regex.Matcher;
 
 import org.apache.commons.lang3.tuple.Pair;
@@ -98,7 +102,6 @@ public class CubingJob extends DefaultChainedExecutable {
 super();
 }
 
-
 void setDeployEnvName(String name) {
 setParam(DEPLOY_ENV_NAME, name);
 }
@@ -126,17 +129,17 @@ public class CubingJob extends DefaultChainedExecutable {
 return null;
 }
 switch (state) {
-case ERROR:
-logMsg = output.getVerboseMsg();
-break;
-case DISCARDED:
-logMsg = "job has been discarded";
-break;
-case SUCCEED:
-logMsg = "job has succeeded";
-break;
-default:
-return null;
+case ERROR:
+logMsg = output.getVerboseMsg();
+break;
+case DISCARDED:
+logMsg = "job has been discarded";
+break;
+case SUCCEED:
+logMsg = "job has succeeded";
+break;
+default:
+return null;
 }
 String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
 content = content.replaceAll("\\$\\{job_name\\}", getName());

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
index 227e176..300b123 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
@@ -119,7 +119,7 @@ public class DFSFileTableReader implements TableReader {
 
 private String[] split(String line, String delim) {
 // FIXME CVS line should be parsed considering escapes
-String str[] = StringSplitter.split(line, delim);
+String[] str = StringSplitter.split(line, delim);
 
 // un-escape CSV
 if (DFSFileTable.DELIM_COMMA.equals(delim)) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
index e989042..276af65 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
@@ -27,7 +27,6 @@ public interface IMROutput {
 /** Return a helper to participate in batch cubing job flow. */
 public IMRBatchCubingOutputSide getBatchCubingOutputSide(CubeSegment seg);
 
-
 /** Return a helper to participate in batch cubing job flow. */
 public IMRBatchInvertedIndexingOutputSide 
getBatchInvertedIndexingOutputSide(IISegment seg);
 
@@ -81,7 +80,6 @@ public interface IMROutput {
 public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow);
 }
 
-
 /**
  * Participate the batch inverted indexing flow as the output side. 
Responsible for saving
  * the output to storage (Phase 3).

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index d7676f1..5228088 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -34,7 +34,6 @@ import 
org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep;
 import org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterMergeStep;
 import org.apache.kylin.job.constant.ExecutableConstants;
 import

[09/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/server/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/ProjectController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
index 980ec8c..f829fff 100644
--- 
a/server/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
+++ 
b/server/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
@@ -73,13 +73,13 @@ public class ProjectController extends BasicController {
  * @return Table metadata array
  * @throws IOException
  */
-@RequestMapping(value = "", method = {RequestMethod.GET})
+@RequestMapping(value = "", method = { RequestMethod.GET })
 @ResponseBody
 public List getProjects(@RequestParam(value = "limit", 
required = false) Integer limit, @RequestParam(value = "offset", required = 
false) Integer offset) {
 return projectService.listAllProjects(limit, offset);
 }
 
-@RequestMapping(value = "/readable", method = {RequestMethod.GET})
+@RequestMapping(value = "/readable", method = { RequestMethod.GET })
 @ResponseBody
 public List getReadableProjects(@RequestParam(value = 
"limit", required = false) Integer limit, @RequestParam(value = "offset", 
required = false) Integer offset) {
 List readableProjects = new 
ArrayList();
@@ -189,12 +189,11 @@ public class ProjectController extends BasicController {
 }
 }
 
-
 }
 return readableProjects;
 }
 
-@RequestMapping(value = "", method = {RequestMethod.POST})
+@RequestMapping(value = "", method = { RequestMethod.POST })
 @ResponseBody
 public ProjectInstance saveProject(@RequestBody CreateProjectRequest 
projectRequest) {
 if (StringUtils.isEmpty(projectRequest.getName())) {
@@ -212,7 +211,7 @@ public class ProjectController extends BasicController {
 return createdProj;
 }
 
-@RequestMapping(value = "", method = {RequestMethod.PUT})
+@RequestMapping(value = "", method = { RequestMethod.PUT })
 @ResponseBody
 public ProjectInstance updateProject(@RequestBody UpdateProjectRequest 
projectRequest) {
 if (StringUtils.isEmpty(projectRequest.getFormerProjectName())) {
@@ -231,7 +230,7 @@ public class ProjectController extends BasicController {
 return updatedProj;
 }
 
-@RequestMapping(value = "/{projectName}", method = {RequestMethod.DELETE})
+@RequestMapping(value = "/{projectName}", method = { RequestMethod.DELETE 
})
 @ResponseBody
 public void deleteProject(@PathVariable String projectName) {
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 88cf912..93b71ad 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -26,10 +26,6 @@ import java.util.List;
 import javax.annotation.PostConstruct;
 import javax.servlet.http.HttpServletResponse;
 
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.debug.BackdoorToggles;
@@ -64,6 +60,10 @@ import org.supercsv.prefs.CsvPreference;
 
 import com.google.common.base.Preconditions;
 
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+
 /**
  * Handle query requests.
  * 

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
 
b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index e305f2c..f3374c3 100644
--- 
a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ 
b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -18,9 +18,10 @@
 
 package org.apache.kylin.rest.controller;
 
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonMappingException;
+import java.io.IOException;
+import java.util.List;
+import java.util.UUID;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import

[11/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/engine-spark/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/engine-spark/.settings/org.eclipse.jdt.core.prefs 
b/engine-spark/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 000..a903301
--- /dev/null
+++ b/engine-spark/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,379 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
+org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
+org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
+org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
+org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
+org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
+org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
+org.eclipse.jdt.core.compiler.problem.deadCode=warning
+org.eclipse.jdt.core.compiler.problem.deprecation=warning
+org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
+org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
+org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
+org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
+org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
+org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
+org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
+org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
+org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
+org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
+org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
+org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
+org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
+org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
+org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
+org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
+org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
+org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
+org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
+org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
+org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
+org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
+org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
+org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
+org.eclipse.jdt.core.compiler.problem.nullReference=warning
+org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
+org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
+org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
+org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
+org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
+org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
+org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore

[15/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
deleted file mode 100644
index d0a07de..000
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/IKylinValidationConstants.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model.validation.rule;
-
-import org.apache.kylin.metadata.MetadataConstants;
-
-public interface IKylinValidationConstants extends MetadataConstants {
-
-public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 12;
-public static final String KEY_MAX_AGR_GROUP_SIZE = 
"rule_max.arggregation.group.size";
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
index 02f084c..4cf0584 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+
 import com.google.common.collect.Lists;
 
 /**

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
index 007b868..ca78d68 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
@@ -66,8 +66,6 @@ public class CubeMetadataUpgrade_v_1_4_0 extends 
CubeMetadataUpgrade {
 //do nothing
 }
 
-  
-
 private CubeDesc loadOldCubeDesc(String path) {
 CubeDesc ndesc = null;
 try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 53cc387..863b958 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -193,8 +193,8 @@ public class GTAggregateScanner implements IGTScanner {
 for (int i = 0; i < types.length; i++) {
 types[i] = info.getColumnType(metrics.trueBitAt(i));
 }
-
-BufferedMeasureEncoder result =  new BufferedMeasureEncoder(types);
+
+BufferedMeasureEncoder result = new BufferedMeasureEncoder(types);
 result.setBufferSize(info.getMaxColumnLength(metrics));
 return result;
 }
@@ -372,7 +372,7 @@ public class GTAggregateScanner implements IGTScanner {
 record.cols[c].set(key, offset, columnLength);
 offset += columnLength;
 }
-
+
 for (int i = 0; i < value.length; i++) {
 tmpValues[i] =

[04/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
index 0e0a8ce..545d058 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileMapper.java
@@ -63,10 +63,9 @@ public class IICreateHFileMapper extends 
KylinMapper

[13/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
index c23d649..021c146 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
@@ -18,15 +18,15 @@
 
 package org.apache.kylin.measure.raw;
 
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
 @SuppressWarnings("unused")
 public class RawSerializer extends DataTypeSerializer {
 
@@ -98,7 +98,7 @@ public class RawSerializer extends 
DataTypeSerializer {
 if (size >= 0) {
 for (int i = 0; i < size; i++) {
 ByteArray ba = new ByteArray(BytesUtil.readByteArray(in));
-if (ba.length() != 0){
+if (ba.length() != 0) {
 values.add(ba);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
index 8dc9e7e..041ea2b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
@@ -18,7 +18,6 @@
 
 package org.apache.kylin.measure.topn;
 
-
 import java.io.Externalizable;
 import java.io.IOException;
 import java.io.ObjectInput;
@@ -33,7 +32,7 @@ public class Counter implements Externalizable {
 
 protected T item;
 protected double count;
-//protected double error;
+//protected double error;
 
 /**
  * For de-serialization
@@ -43,25 +42,25 @@ public class Counter implements Externalizable {
 
 public Counter(T item) {
 this.count = 0;
-//this.error = 0;
+//this.error = 0;
 this.item = item;
 }
 
 public T getItem() {
 return item;
 }
-
+
 public double getCount() {
 return count;
 }
 
-//public double getError() {
-//return error;
-//}
+//public double getError() {
+//return error;
+//}
 
 @Override
 public String toString() {
-//return item + ":" + count + ':' + error;
+//return item + ":" + count + ':' + error;
 return item + ":" + count;
 }
 
@@ -80,4 +79,3 @@ public class Counter implements Externalizable {
 //out.writeDouble(error);
 }
 }
-

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
index 2d9e3ee..38d5b20 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
@@ -30,7 +30,7 @@ public class DoubleDeltaSerializer {
 static final int PRECISION_BITS = 3;
 static final int DELTA_SIZE_BITS = 6;
 static final int LENGTH_BITS = 23;
-
+
 static final long[] MASKS = new long[64];
 {
 for (int i = 0; i < MASKS.length; i++) {
@@ -41,7 +41,6 @@ public class DoubleDeltaSerializer {
 final private int precision;
 final private int multiplier;
 
-
 transient ThreadLocal deltasThreadLocal;
 
 public DoubleDeltaSerializer() {
@@ -59,7 +58,7 @@ public class DoubleDeltaSerializer {
 this.deltasThreadLocal = new ThreadLocal();
 }
 
-public void serialize(double values[], ByteBuffer buf) {
+public void serialize(double[] values, ByteBuffer buf) {
 long[] deltas = calculateDeltas(values);
 int deltaSize = maxDeltaSize(deltas, values.length - 1);
 
@@ -118,7 +117,7 @@ public class DoubleDeltaSerializer {
 deltas = new long[len];
 deltasThreadLocal.set(deltas);
 }
-
+
 if (len == 0)
 return deltas;
 
@@ -144,17 +143,17 @@ public class

[18/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code 
formatting


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d4982e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d4982e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d4982e2

Branch: refs/heads/master
Commit: 5d4982e247a2172d97d44c85309cef4b3dbfce09
Parents: f26f702
Author: Hongbin Ma 
Authored: Fri Jun 24 16:19:05 2016 +0800
Committer: Hongbin Ma 
Committed: Sat Jun 25 19:10:05 2016 +0800

--
 .../.settings/org.eclipse.core.resources.prefs  |5 +
 assembly/.settings/org.eclipse.jdt.core.prefs   |  379 +
 assembly/.settings/org.eclipse.jdt.ui.prefs |7 +
 .../kylin/job/DeployLocalMetaToRemoteTest.java  |1 -
 .../java/org/apache/kylin/job/DeployUtil.java   |6 +-
 .../kylin/job/hadoop/invertedindex/IITest.java  |   13 +-
 .../kylin/job/streaming/KafkaDataLoader.java|8 +-
 .../streaming/StreamingTableDataGenerator.java  |1 +
 .../calcite/sql2rel/SqlToRelConverter.java  | 8302 -
 checkstyle-suppressions.xml |   37 +
 checkstyle.xml  |   60 +
 .../org/apache/kylin/common/KylinConfig.java|   32 +-
 .../common/persistence/FileResourceStore.java   |2 +-
 .../common/persistence/JsonSerializer.java  |2 +-
 .../persistence/RootPersistentEntity.java   |   13 -
 .../common/restclient/AbstractRestCache.java|2 +-
 .../kylin/common/restclient/Broadcaster.java|2 +-
 .../common/restclient/SingleValueCache.java |6 +-
 .../apache/kylin/common/util/BytesSplitter.java |   15 +-
 .../org/apache/kylin/common/util/BytesUtil.java |4 +-
 .../apache/kylin/common/util/Dictionary.java|6 +-
 .../kylin/common/util/ImmutableBitSet.java  |2 +-
 .../kylin/common/util/ImplementationSwitch.java |5 +-
 .../common/util/LoggableCachedThreadPool.java   |4 +-
 .../common/util/MemoryBudgetController.java |3 +-
 .../kylin/common/util/MyLogFormatter.java   |1 +
 .../org/apache/kylin/common/util/Primes.java|   31 +-
 .../org/apache/kylin/common/util/RangeUtil.java |6 +-
 .../org/apache/kylin/common/util/SSHClient.java |   13 +-
 .../apache/kylin/common/util/StringUtil.java|4 +-
 .../apache/kylin/common/KylinConfigTest.java|3 +-
 .../common/persistence/ResourceStoreTest.java   |   18 +-
 .../common/util/AbstractKylinTestCase.java  |1 -
 .../kylin/common/util/DateFormatTest.java   |5 +-
 .../common/util/HBaseMetadataTestCase.java  |3 +-
 .../apache/kylin/common/util/RangeUtilTest.java |1 -
 .../org/apache/kylin/cube/CubeInstance.java |3 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |   13 +-
 .../org/apache/kylin/cube/CubeValidator.java|2 +-
 .../kylin/cube/cli/CubeSignatureRefresher.java  |4 +-
 .../kylin/cube/cli/DictionaryGeneratorCLI.java  |4 +-
 .../kylin/cube/common/RowKeySplitter.java   |1 -
 .../org/apache/kylin/cube/cuboid/Cuboid.java|5 +-
 .../kylin/cube/gridtable/CubeGridTable.java |2 +-
 .../gridtable/CuboidToGridTableMapping.java |   12 +-
 .../kylin/cube/gridtable/TrimmedDimEnc.java |6 +-
 .../gridtable/TrimmedDimensionSerializer.java   |4 +-
 .../inmemcubing/AbstractInMemCubeBuilder.java   |2 +-
 .../cube/inmemcubing/CompoundCuboidWriter.java  |4 +-
 .../cube/inmemcubing/ConcurrentDiskStore.java   |4 +-
 .../kylin/cube/inmemcubing/ICuboidWriter.java   |6 +-
 .../cube/inmemcubing/InMemCubeBuilder.java  |6 +-
 .../InMemCubeBuilderInputConverter.java |   23 +-
 .../cube/inmemcubing/InMemCubeBuilderUtils.java |4 +-
 .../kylin/cube/inmemcubing/MemDiskStore.java|4 +-
 .../kylin/cube/model/AggregationGroup.java  |3 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   23 +-
 .../apache/kylin/cube/model/DictionaryDesc.java |   12 +-
 .../apache/kylin/cube/model/DimensionDesc.java  |2 +-
 .../kylin/cube/model/HBaseColumnDesc.java   |4 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |3 +-
 .../org/apache/kylin/cube/model/RowKeyDesc.java |5 +-
 .../kylin/cube/model/v1_4_0/CubeDesc.java   |   12 +-
 .../cube/model/v1_4_0/HBaseColumnDesc.java  |4 +-
 .../model/v1_4_0/HBaseColumnFamilyDesc.java |1 -
 .../kylin/cube/model/v1_4_0/RowKeyDesc.java |4 +-
 .../model/validation/CubeMetadataValidator.java |2 +-
 .../cube/model/validation/IValidatorRule.java   |5 +-
 .../model/validation/rule/DictionaryRule.java   |   11 +-
 .../model/validation/rule/FunctionRule.java |3 +-
 .../rule/IKylinValidationConstants.java |   28 -
 .../common/MetadataVersionRefresher.java|1 +
 .../v1_4_0/CubeMetadataUpgrade_v_1_4_0.java |2 -

[07/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
index 6266f17..1bd92e6 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
@@ -22,1330 +22,1277 @@
 package org.apache.kylin.storage.hbase.cube.v1.filter.generated;
 
 public final class FilterProtosExt {
-  private FilterProtosExt() {}
-  public static void registerAllExtensions(
-  com.google.protobuf.ExtensionRegistry registry) {
-  }
-  public interface BytesBytesPairOrBuilder
-  extends com.google.protobuf.MessageOrBuilder {
-
-// required bytes first = 1;
-/**
- * required bytes first = 1;
- */
-boolean hasFirst();
-/**
- * required bytes first = 1;
- */
-com.google.protobuf.ByteString getFirst();
-
-// required bytes second = 2;
-/**
- * required bytes second = 2;
- */
-boolean hasSecond();
-/**
- * required bytes second = 2;
- */
-com.google.protobuf.ByteString getSecond();
-  }
-  /**
-   * Protobuf type {@code BytesBytesPair}
-   */
-  public static final class BytesBytesPair extends
-  com.google.protobuf.GeneratedMessage
-  implements BytesBytesPairOrBuilder {
-// Use BytesBytesPair.newBuilder() to construct.
-private BytesBytesPair(com.google.protobuf.GeneratedMessage.Builder 
builder) {
-  super(builder);
-  this.unknownFields = builder.getUnknownFields();
+private FilterProtosExt() {
 }
-private BytesBytesPair(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
 
-private static final BytesBytesPair defaultInstance;
-public static BytesBytesPair getDefaultInstance() {
-  return defaultInstance;
+public static void 
registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
 }
 
-public BytesBytesPair getDefaultInstanceForType() {
-  return defaultInstance;
-}
+public interface BytesBytesPairOrBuilder extends 
com.google.protobuf.MessageOrBuilder {
 
-private final com.google.protobuf.UnknownFieldSet unknownFields;
-@java.lang.Override
-public final com.google.protobuf.UnknownFieldSet
-getUnknownFields() {
-  return this.unknownFields;
-}
-private BytesBytesPair(
-com.google.protobuf.CodedInputStream input,
-com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-throws com.google.protobuf.InvalidProtocolBufferException {
-  initFields();
-  int mutable_bitField0_ = 0;
-  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
-  com.google.protobuf.UnknownFieldSet.newBuilder();
-  try {
-boolean done = false;
-while (!done) {
-  int tag = input.readTag();
-  switch (tag) {
-case 0:
-  done = true;
-  break;
-default: {
-  if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
-done = true;
-  }
-  break;
-}
-case 10: {
-  bitField0_ |= 0x0001;
-  first_ = input.readBytes();
-  break;
-}
-case 18: {
-  bitField0_ |= 0x0002;
-  second_ = input.readBytes();
-  break;
-}
-  }
-}
-  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-throw e.setUnfinishedMessage(this);
-  } catch (java.io.IOException e) {
-throw new com.google.protobuf.InvalidProtocolBufferException(
-e.getMessage()).setUnfinishedMessage(this);
-  } finally {
-this.unknownFields = unknownFields.build();
-makeExtensionsImmutable();
-  }
-}
-public static final com.google.protobuf.Descriptors.Descriptor
-getDescriptor() {
-  return 
org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_BytesBytesPair_descriptor;
-}
+// required bytes first = 1;
+/**
+ * required bytes first = 1;
+ */
+boolean hasFirst();
 
-protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-internalGetFieldAccessorTable() {
-  return 
org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_BytesBytesPair_fieldAccessorTable
-  .ensureFieldAccessorsInitialized(
-

[01/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master f26f702a6 -> 5d4982e24


http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/tool/.settings/org.eclipse.jdt.core.prefs
--
diff --git a/tool/.settings/org.eclipse.jdt.core.prefs 
b/tool/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 000..a903301
--- /dev/null
+++ b/tool/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,379 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled
+org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
+org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
+org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
+org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
+org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
+org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
+org.eclipse.jdt.core.compiler.problem.deadCode=warning
+org.eclipse.jdt.core.compiler.problem.deprecation=warning
+org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
+org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
+org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
+org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
+org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
+org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
+org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
+org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
+org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
+org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
+org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
+org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
+org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
+org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
+org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
+org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
+org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
+org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
+org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
+org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
+org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
+org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
+org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
+org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
+org.eclipse.jdt.core.compiler.problem.nullReference=warning
+org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
+org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning
+org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
+org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
+org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
+org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore

[10/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index 59bdc2d..465a983 100644
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -45,7 +45,9 @@ import java.util.concurrent.LinkedBlockingQueue;
 import javax.annotation.Nullable;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.StreamingBatch;
+import org.apache.kylin.common.util.StreamingMessage;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
@@ -53,10 +55,8 @@ import org.apache.kylin.cube.CubeUpdate;
 import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
 import org.apache.kylin.cube.inmemcubing.InMemCubeBuilder;
 import org.apache.kylin.cube.util.CubingUtils;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.StreamingBatch;
 import org.apache.kylin.engine.streaming.StreamingBatchBuilder;
-import org.apache.kylin.common.util.StreamingMessage;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -91,7 +91,7 @@ public class StreamingCubeBuilder implements 
StreamingBatchBuilder {
 for (StreamingMessage streamingMessage : 
streamingBatch.getMessages()) {
 blockingQueue.put(streamingMessage.getData());
 }
-blockingQueue.put(Collections.emptyList());
+blockingQueue.put(Collections. emptyList());
 future.get();
 cuboidWriter.flush();
 
@@ -148,18 +148,14 @@ public class StreamingCubeBuilder implements 
StreamingBatchBuilder {
 final CubeInstance cubeInstance = 
cubeManager.reloadCubeLocal(cubeName);
 final Map dictionaryMap;
 try {
-dictionaryMap = CubingUtils.buildDictionary(cubeInstance,
-Lists.transform(streamingBatch.getMessages(), new 
Function() {
-@Nullable
-@Override
-public List apply(@Nullable StreamingMessage 
input) {
-return input.getData();
-}
-}));
-Map realDictMap = 
CubingUtils.writeDictionary((CubeSegment) buildable,
-dictionaryMap,
-streamingBatch.getTimeRange().getFirst(),
-streamingBatch.getTimeRange().getSecond());
+dictionaryMap = CubingUtils.buildDictionary(cubeInstance, 
Lists.transform(streamingBatch.getMessages(), new Function() {
+@Nullable
+@Override
+public List apply(@Nullable StreamingMessage input) {
+return input.getData();
+}
+}));
+Map realDictMap = 
CubingUtils.writeDictionary((CubeSegment) buildable, dictionaryMap, 
streamingBatch.getTimeRange().getFirst(), 
streamingBatch.getTimeRange().getSecond());
 return realDictMap;
 } catch (IOException e) {
 throw new RuntimeException("failed to build dictionary", e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
index 66a0af2..4038e53 100644
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
+++ 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/util/StreamingUtils.java
@@ -33,17 +33,14 @@
  */
 package org.apache.kylin.engine.streaming.util;
 
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.engine.streaming.IStreamingInput;
 import org.apache.kylin.engine.streaming.IStreamingOutput;
 import org.apache.kylin.engine.streaming.StreamingBatchBuilder;
-import

[03/18] kylin git commit: KYLIN-1821 Reformat all of the java files and enable checkstyle to enforce code formatting

2016-06-25 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/5d4982e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
index 916a0a0..e3f9ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/generated/IIProtos.java
@@ -22,3895 +22,3752 @@
 package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated;
 
 public final class IIProtos {
-  private IIProtos() {}
-  public static void registerAllExtensions(
-  com.google.protobuf.ExtensionRegistry registry) {
-  }
-  public interface IIRequestOrBuilder
-  extends com.google.protobuf.MessageOrBuilder {
-
-// required bytes type = 1;
-/**
- * required bytes type = 1;
- */
-boolean hasType();
-/**
- * required bytes type = 1;
- */
-com.google.protobuf.ByteString getType();
-
-// required bytes filter = 2;
-/**
- * required bytes filter = 2;
- */
-boolean hasFilter();
-/**
- * required bytes filter = 2;
- */
-com.google.protobuf.ByteString getFilter();
-
-// required bytes projector = 3;
-/**
- * required bytes projector = 3;
- */
-boolean hasProjector();
-/**
- * required bytes projector = 3;
- */
-com.google.protobuf.ByteString getProjector();
-
-// required bytes aggregator = 4;
-/**
- * required bytes aggregator = 4;
- */
-boolean hasAggregator();
-/**
- * required bytes aggregator = 4;
- */
-com.google.protobuf.ByteString getAggregator();
-
-// optional bytes tsRange = 5;
-/**
- * optional bytes tsRange = 5;
- */
-boolean hasTsRange();
-/**
- * optional bytes tsRange = 5;
- */
-com.google.protobuf.ByteString getTsRange();
-  }
-  /**
-   * Protobuf type {@code IIRequest}
-   */
-  public static final class IIRequest extends
-  com.google.protobuf.GeneratedMessage
-  implements IIRequestOrBuilder {
-// Use IIRequest.newBuilder() to construct.
-private IIRequest(com.google.protobuf.GeneratedMessage.Builder builder) 
{
-  super(builder);
-  this.unknownFields = builder.getUnknownFields();
-}
-private IIRequest(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-private static final IIRequest defaultInstance;
-public static IIRequest getDefaultInstance() {
-  return defaultInstance;
-}
-
-public IIRequest getDefaultInstanceForType() {
-  return defaultInstance;
-}
-
-private final com.google.protobuf.UnknownFieldSet unknownFields;
-@java.lang.Override
-public final com.google.protobuf.UnknownFieldSet
-getUnknownFields() {
-  return this.unknownFields;
-}
-private IIRequest(
-com.google.protobuf.CodedInputStream input,
-com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-throws com.google.protobuf.InvalidProtocolBufferException {
-  initFields();
-  int mutable_bitField0_ = 0;
-  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
-  com.google.protobuf.UnknownFieldSet.newBuilder();
-  try {
-boolean done = false;
-while (!done) {
-  int tag = input.readTag();
-  switch (tag) {
-case 0:
-  done = true;
-  break;
-default: {
-  if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
-done = true;
-  }
-  break;
-}
-case 10: {
-  bitField0_ |= 0x0001;
-  type_ = input.readBytes();
-  break;
-}
-case 18: {
-  bitField0_ |= 0x0002;
-  filter_ = input.readBytes();
-  break;
-}
-case 26: {
-  bitField0_ |= 0x0004;
-  projector_ = input.readBytes();
-  break;
-}
-case 34: {
-  bitField0_ |= 0x0008;
-  aggregator_ = input.readBytes();
-  break;
-}
-case 42: {
-  bitField0_ |= 0x0010;
-  tsRange_ = input.readBytes();
-  break;
-}
-  }
-}
-  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-throw e.setUnfinishedMessage(this);
-  } catch (java.io.IOException e) {
-throw new

kylin git commit: minor: disable printing stacktrace when create a KylinConfig instance

2016-06-23 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master c7439a0a4 -> 842f590af


minor: disable printing stacktrace when create a KylinConfig instance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/842f590a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/842f590a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/842f590a

Branch: refs/heads/master
Commit: 842f590af8af2c95d08e7409ea719ef62b18807e
Parents: c7439a0
Author: Hongbin Ma 
Authored: Fri Jun 24 09:52:21 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Jun 24 09:52:26 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java| 31 +++-
 1 file changed, 17 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/842f590a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 9eebde7..8719541 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -371,19 +371,22 @@ public class KylinConfig extends KylinConfigBase {
 }
 
 private static void dumpStackTrace() {
-Thread t = Thread.currentThread();
-int maxStackTraceDepth = 20;
-int current = 0;
-
-StackTraceElement[] stackTrace = t.getStackTrace();
-StringBuilder buf = new StringBuilder("This is not a exception, just 
for diagnose purpose:");
-buf.append("\n");
-for (StackTraceElement e : stackTrace) {
-if (++current > maxStackTraceDepth) {
-break;
-}
-buf.append("\t").append("at ").append(e.toString()).append("\n");
-}
-logger.info(buf.toString());
+
+//uncomment below to start debugging
+
+//Thread t = Thread.currentThread();
+//int maxStackTraceDepth = 20;
+//int current = 0;
+//
+//StackTraceElement[] stackTrace = t.getStackTrace();
+//StringBuilder buf = new StringBuilder("This is not a exception, just 
for diagnose purpose:");
+//buf.append("\n");
+//for (StackTraceElement e : stackTrace) {
+//if (++current > maxStackTraceDepth) {
+//break;
+//}
+//buf.append("\t").append("at ").append(e.toString()).append("\n");
+//}
+//logger.info(buf.toString());
 }
 }

kylin git commit: refactor about default gtstore

2016-06-23 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 62ae3cb16 -> 4fd74fc6a


refactor about default gtstore


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4fd74fc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4fd74fc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4fd74fc6

Branch: refs/heads/master
Commit: 4fd74fc6a7d7c9bcccab38b57da76f2b983cf5ef
Parents: 62ae3cb
Author: Hongbin Ma 
Authored: Thu Jun 23 16:37:09 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jun 23 16:37:09 2016 +0800

--
 .../apache/kylin/gridtable/ScannerWorker.java   |  11 +-
 .../storage/gtrecord/CubeSegmentScanner.java| 100 +
 .../storage/gtrecord/CubeTupleConverter.java| 270 +
 .../gtrecord/GTCubeStorageQueryBase.java| 377 +++
 .../gtrecord/SequentialCubeTupleIterator.java   | 210 +++
 .../hbase/cube/v2/CubeSegmentScanner.java   | 100 -
 .../storage/hbase/cube/v2/CubeStorageQuery.java | 349 +
 .../hbase/cube/v2/CubeTupleConverter.java   | 270 -
 .../cube/v2/SequentialCubeTupleIterator.java| 210 ---
 9 files changed, 968 insertions(+), 929 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd74fc6/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
index 1ac3b02..586a584 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
@@ -24,8 +24,6 @@ import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.Iterator;
 
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.slf4j.Logger;
@@ -36,7 +34,7 @@ public class ScannerWorker {
 private static final Logger logger = 
LoggerFactory.getLogger(ScannerWorker.class);
 private IGTScanner internal = null;
 
-public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest 
scanRequest) {
+public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest 
scanRequest, String gtStorage) {
 if (scanRequest == null) {
 logger.info("Segment {} will be skipped", cubeSeg);
 internal = new EmptyGTScanner(0);
@@ -46,12 +44,7 @@ public class ScannerWorker {
 final GTInfo info = scanRequest.getInfo();
 
 try {
-IGTStorage rpc;
-if 
("scan".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryProtocol())) {
-rpc = (IGTStorage) 
Class.forName("org.apache.kylin.storage.hbase.cube.v2.CubeHBaseScanRPC").getConstructor(CubeSegment.class,
 Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // for local 
debug
-} else {
-rpc = (IGTStorage) 
Class.forName(KylinConfig.getInstanceFromEnv().getDefaultIGTStorage()).getConstructor(CubeSegment.class,
 Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // default 
behavior
-}
+IGTStorage rpc = (IGTStorage) 
Class.forName(gtStorage).getConstructor(CubeSegment.class, Cuboid.class, 
GTInfo.class).newInstance(cubeSeg, cuboid, info); // default behavior
 internal = rpc.getGTScanner(scanRequest);
 } catch (IOException | InstantiationException | 
InvocationTargetException | IllegalAccessException | ClassNotFoundException | 
NoSuchMethodException e) {
 throw new RuntimeException("error", e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd74fc6/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
new file mode 100644
index 000..c12159d
--- /dev/null
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.

kylin git commit: KYLIN-1816 More than one base KylinConfig exist in spring JVM

2016-06-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 1b0ece4a4 -> 62ae3cb16


KYLIN-1816 More than one base KylinConfig exist in spring JVM


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62ae3cb1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62ae3cb1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62ae3cb1

Branch: refs/heads/master
Commit: 62ae3cb165126404b1ef95009ff1095fe39e2441
Parents: 1b0ece4
Author: Hongbin Ma 
Authored: Thu Jun 23 12:04:32 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jun 23 13:58:17 2016 +0800

--
 core-common/src/main/java/org/apache/kylin/common/KylinConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/62ae3cb1/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index a4024ae..9eebde7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -376,7 +376,7 @@ public class KylinConfig extends KylinConfigBase {
 int current = 0;
 
 StackTraceElement[] stackTrace = t.getStackTrace();
-StringBuilder buf = new StringBuilder("");
+StringBuilder buf = new StringBuilder("This is not a exception, just 
for diagnose purpose:");
 buf.append("\n");
 for (StackTraceElement e : stackTrace) {
 if (++current > maxStackTraceDepth) {

kylin git commit: KYLIN-1816 More than one base KylinConfig exist in spring JVM

2016-06-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master b514ae16a -> 1b0ece4a4


KYLIN-1816 More than one base KylinConfig exist in spring JVM


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b0ece4a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b0ece4a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b0ece4a

Branch: refs/heads/master
Commit: 1b0ece4a433ca3417d13a6d5fc7b5c1e503e341b
Parents: b514ae1
Author: Hongbin Ma 
Authored: Thu Jun 23 12:04:32 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jun 23 12:04:43 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java| 78 +---
 .../org/apache/kylin/common/KylinConfigExt.java |  5 ++
 .../kylin/common/restclient/Broadcaster.java|  2 +-
 .../common/util/SerializeToByteBuffer.java  | 44 +++
 .../java/org/apache/kylin/cube/CubeManager.java |  9 ++-
 .../apache/kylin/gridtable/GTScanRequest.java   | 13 +++-
 .../apache/kylin/dict/DictionaryManager.java|  3 +
 .../kylin/dict/lookup/SnapshotManager.java  | 21 +++---
 .../org/apache/kylin/job/dao/ExecutableDao.java |  3 +
 .../kylin/job/execution/AbstractExecutable.java |  2 +-
 .../kylin/job/manager/ExecutableManager.java|  3 +
 .../realization/RealizationRegistry.java|  3 +
 .../kylin/storage/hybrid/HybridManager.java |  2 +-
 .../engine/streaming/StreamingManager.java  |  2 +-
 .../kylin/source/kafka/KafkaConfigManager.java  |  2 +-
 15 files changed, 147 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index b4b3c1b..a4024ae 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -61,22 +61,28 @@ public class KylinConfig extends KylinConfigBase {
 private static KylinConfig ENV_INSTANCE = null;
 
 public static KylinConfig getInstanceFromEnv() {
-if (ENV_INSTANCE == null) {
-try {
-KylinConfig config = loadKylinConfig();
-ENV_INSTANCE = config;
-} catch (IllegalArgumentException e) {
-throw new IllegalStateException("Failed to find KylinConfig ", 
e);
+synchronized (KylinConfig.class) {
+if (ENV_INSTANCE == null) {
+try {
+KylinConfig config = loadKylinConfig();
+logger.info("Initialized a new KylinConfig from 
getInstanceFromEnv : " + System.identityHashCode(config));
+ENV_INSTANCE = config;
+} catch (IllegalArgumentException e) {
+throw new IllegalStateException("Failed to find 
KylinConfig ", e);
+}
 }
+return ENV_INSTANCE;
 }
-return ENV_INSTANCE;
 }
 
+//Only used in test cases!!! 
 public static void destroyInstance() {
+logger.info("Destory KylinConfig");
+dumpStackTrace();
 ENV_INSTANCE = null;
 }
 
-public static enum UriType {
+public enum UriType {
 PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER
 }
 
@@ -150,12 +156,15 @@ public class KylinConfig extends KylinConfigBase {
 }
 
 public static void setKylinConfigFromInputStream(InputStream is) {
-if (ENV_INSTANCE == null) {
-try {
-KylinConfig config = createKylinConfigFromInputStream(is);
-ENV_INSTANCE = config;
-} catch (IllegalArgumentException e) {
-throw new IllegalStateException("Failed to find KylinConfig ", 
e);
+synchronized (KylinConfig.class) {
+if (ENV_INSTANCE == null) {
+try {
+KylinConfig config = createKylinConfigFromInputStream(is);
+logger.info("Resetting ENV_INSTANCE by a input stream: " + 
System.identityHashCode(config));
+ENV_INSTANCE = config;
+} catch (IllegalArgumentException e) {
+throw new IllegalStateException("Failed to find 
KylinConfig ", e);
+}
 }
 }
 }
@@ -254,11 +263,11 @@ public class KylinConfig extends KylinConfigBase {
 
 return config;
 }
-
+
 public static void setSandboxEnvIfPossible() {
 File dir1 = new File("../examples/test_case_data/sandbox");
 File dir2 = new File("../../kylin/examples/test_case_data/sandbox");
-
+

[1/2] kylin git commit: refactor: more checks on finding segment by id

2016-06-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 9b8a9899b -> 91d6798d4


refactor: more checks on finding segment by id


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c736ee5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c736ee5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c736ee5f

Branch: refs/heads/master
Commit: c736ee5fe8009c28f5e016d24c5f113940bc11cf
Parents: 9b8a989
Author: Hongbin Ma 
Authored: Wed Jun 22 19:06:17 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 19:06:41 2016 +0800

--
 .../engine/mr/steps/SaveStatisticsStep.java | 48 ++--
 1 file changed, 43 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c736ee5f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 435fb13..7cc9dc3 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -21,6 +21,9 @@ package org.apache.kylin.engine.mr.steps;
 import java.io.IOException;
 import java.util.Random;
 
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -43,9 +46,11 @@ import org.apache.kylin.metadata.model.MeasureDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+
 /**
  * Save the cube segment statistic to Kylin metadata store
- *
  */
 public class SaveStatisticsStep extends AbstractExecutable {
 
@@ -55,12 +60,45 @@ public class SaveStatisticsStep extends AbstractExecutable {
 super();
 }
 
+private CubeSegment findSegment(ExecutableContext context, String 
cubeName, String segmentId) {
+final CubeManager mgr = CubeManager.getInstance(context.getConfig());
+final CubeInstance cube = mgr.getCube(cubeName);
+
+if (cube == null) {
+String cubeList = 
StringUtils.join(Iterables.transform(mgr.listAllCubes(), new 
Function() {
+@Nullable
+@Override
+public String apply(@Nullable CubeInstance input) {
+return input.getName();
+}
+}).iterator(), ",");
+
+logger.info("target cube name: {}, cube list: {}", cubeName, 
cubeList);
+throw new IllegalStateException();
+}
+
+
+final CubeSegment newSegment = cube.getSegmentById(segmentId);
+
+if (newSegment == null) {
+String segmentList = 
StringUtils.join(Iterables.transform(cube.getSegments(), new 
Function() {
+@Nullable
+@Override
+public String apply(@Nullable CubeSegment input) {
+return input.getUuid();
+}
+}).iterator(), ",");
+
+logger.info("target segment id: {}, segment list: {}", segmentId, 
segmentList);
+throw new IllegalStateException();
+}
+return newSegment;
+}
+
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
-final CubeManager mgr = CubeManager.getInstance(context.getConfig());
-final CubeInstance cube = 
mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
-final CubeSegment newSegment = 
cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
-KylinConfig kylinConf = cube.getConfig();
+CubeSegment newSegment = findSegment(context, 
CubingExecutableUtil.getCubeName(this.getParams()), 
CubingExecutableUtil.getSegmentId(this.getParams()));
+KylinConfig kylinConf = newSegment.getConfig();
 
 ResourceStore rs = ResourceStore.getStore(kylinConf);
 try {

[2/2] kylin git commit: fix compile

2016-06-22 Thread mahongbin

fix compile


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/91d6798d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/91d6798d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/91d6798d

Branch: refs/heads/master
Commit: 91d6798d47db0e4c3f20578d8d81f4390b082fdb
Parents: c736ee5
Author: Hongbin Ma 
Authored: Wed Jun 22 19:06:35 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 19:07:24 2016 +0800

--
 .../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/91d6798d/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 593469a..d520efa 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -63,7 +63,7 @@ public class GTScanRequest {
 this(info, ranges, dimensions, aggrGroupBy, aggrMetrics, 
aggrMetricsFuncs, filterPushDown, true, 0);
 }
 
-private GTScanRequest(GTInfo info, List ranges, 
ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
+public GTScanRequest(GTInfo info, List ranges, 
ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
 ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, 
TupleFilter filterPushDown, boolean allowPreAggregation, double aggrCacheGB) {
 this.info = info;
 if (ranges == null) {

kylin git commit: refactor: subclass of AbstractExecutable change to use own logger

2016-06-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master bfee5ed44 -> 94567c150


refactor: subclass of AbstractExecutable change to use own logger


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94567c15
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94567c15
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94567c15

Branch: refs/heads/master
Commit: 94567c150e94d0f6499bb26621869c83d117c5a4
Parents: bfee5ed
Author: Hongbin Ma 
Authored: Wed Jun 22 17:51:55 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 17:52:03 2016 +0800

--
 .../kylin/job/common/ShellExecutable.java   |  4 
 .../kylin/job/execution/AbstractExecutable.java |  2 +-
 .../org/apache/kylin/engine/mr/CubingJob.java   |  4 
 .../engine/mr/common/HadoopShellExecutable.java |  4 
 .../UpdateIIInfoAfterBuildStep.java |  4 
 .../engine/mr/steps/MergeDictionaryStep.java|  5 
 .../engine/mr/steps/MergeStatisticsStep.java|  4 
 .../engine/mr/steps/SaveStatisticsStep.java |  4 
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  4 
 .../mr/steps/UpdateCubeInfoAfterMergeStep.java  |  4 
 .../kylin/engine/spark/SparkExecutable.java |  4 
 .../source/hive/CreateFlatHiveTableStep.java|  4 
 .../apache/kylin/source/hive/HiveMRInput.java   | 25 +++-
 .../apache/kylin/source/hive/HqlExecutable.java |  4 
 .../steps/HDFSPathGarbageCollectionStep.java|  5 
 15 files changed, 69 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 74f54a1..209d14e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -32,11 +32,15 @@ import org.apache.kylin.job.execution.ExecutableContext;
 import org.apache.kylin.job.execution.ExecuteResult;
 
 import com.google.common.collect.Maps;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public class ShellExecutable extends AbstractExecutable {
 
+private static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(ShellExecutable.class);
+
+
 private static final String CMD = "cmd";
 
 public ShellExecutable() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 395676b..778763c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -49,7 +49,7 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 protected static final String START_TIME = "startTime";
 protected static final String END_TIME = "endTime";
 
-protected static final Logger logger = 
LoggerFactory.getLogger(AbstractExecutable.class);
+private static final Logger logger = 
LoggerFactory.getLogger(AbstractExecutable.class);
 protected int retry = 0;
 
 private String name;

http://git-wip-us.apache.org/repos/asf/kylin/blob/94567c15/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 70b6132..bbd0fc5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -42,11 +42,15 @@ import org.apache.kylin.job.execution.ExecuteResult;
 import org.apache.kylin.job.execution.Output;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public class CubingJob extends DefaultChainedExecutable {
 
+private static final Logger logger = 
LoggerFactory.getLogger(CubingJob.class);
+
 public enum AlgorithmEnum {
 LAYER, INMEM
 }

kylin git commit: fix ci

2016-06-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 5fd26125b -> 97bfd5747


fix ci


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/97bfd574
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/97bfd574
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/97bfd574

Branch: refs/heads/master
Commit: 97bfd574796047b5a7e818f17620bae1f67e27f3
Parents: 5fd2612
Author: Hongbin Ma 
Authored: Wed Jun 22 14:47:17 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 14:47:36 2016 +0800

--
 .../java/org/apache/kylin/gridtable/DictGridTableTest.java   | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/97bfd574/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 1687a4f..a328216 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -119,7 +119,7 @@ public class DictGridTableTest extends 
LocalFileMetadataTestCase {
 assertEquals(1, r.size());//scan range are [close,close]
 assertEquals("[null, 10]-[142119360, 10]", 
r.get(0).toString());
 assertEquals(1, r.get(0).fuzzyKeys.size());
-assertEquals("[[10]]", r.get(0).fuzzyKeys.toString());
+assertEquals("[[null, 10, null, null, null]]", 
r.get(0).fuzzyKeys.toString());
 }
 {
 LogicalTupleFilter filter = and(timeComp2, ageComp1);
@@ -145,7 +145,7 @@ public class DictGridTableTest extends 
LocalFileMetadataTestCase {
 List r = planner.planScanRanges();
 assertEquals(1, r.size());
 assertEquals("[142119360, 10]-[null, 10]", 
r.get(0).toString());
-assertEquals("[[10], [142119360, 10]]", 
r.get(0).fuzzyKeys.toString());
+assertEquals("[[null, 10, null, null, null], [142119360, 10, 
null, null, null]]", r.get(0).fuzzyKeys.toString());
 }
 {
 LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6);
@@ -191,7 +191,7 @@ public class DictGridTableTest extends 
LocalFileMetadataTestCase {
 assertEquals(1, r.size());//scan range are [close,close]
 assertEquals("[null, 10]-[142119360, 10]", 
r.get(0).toString());
 assertEquals(1, r.get(0).fuzzyKeys.size());
-assertEquals("[[10]]", r.get(0).fuzzyKeys.toString());
+assertEquals("[[null, 10, null, null, null]]", 
r.get(0).fuzzyKeys.toString());
 }
 
 {
@@ -212,7 +212,7 @@ public class DictGridTableTest extends 
LocalFileMetadataTestCase {
 List r = planner.planScanRanges();
 assertEquals(1, r.size());
 assertEquals("[142119360, 10]-[null, 20]", 
r.get(0).toString());
-assertEquals("[[10], [20]]", r.get(0).fuzzyKeys.toString());
+assertEquals("[[null, 10, null, null, null], [null, 20, null, 
null, null]]", r.get(0).fuzzyKeys.toString());
 }
 
 // pre-evaluate ever false

[1/3] kylin git commit: refactor: minor changes

2016-06-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master ff1c81395 -> b361d8aaa


refactor: minor changes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b361d8aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b361d8aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b361d8aa

Branch: refs/heads/master
Commit: b361d8aaa2b1b5d0d86107bedb9f34397be000a5
Parents: ea24450
Author: Hongbin Ma 
Authored: Wed Jun 22 13:59:58 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Jun 22 14:00:15 2016 +0800

--
 .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 21 ++-
 .../org/apache/kylin/gridtable/GTRecord.java| 17 +-
 .../gridtable/memstore/GTSimpleMemStore.java| 20 +-
 .../kylin/gridtable/DictGridTableTest.java  | 64 ++--
 .../coprocessor/endpoint/CubeVisitService.java  |  2 +-
 5 files changed, 70 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index db704dc..ebcbadd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -94,12 +94,31 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
 @Override
 public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] 
buf) {
 ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0);
-record.exportColumns(keyColumns, byteArray, defaultValue());
+
+encodeDims(record, keyColumns, byteArray, defaultValue());
 
 //fill shard and cuboid
 fillHeader(buf);
 }
 
+//ByteArray representing dimension does not have extra header
+public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, 
ByteArray buf, byte defaultValue) {
+int pos = 0;
+for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+int c = selectedCols.trueBitAt(i);
+ByteArray columnC = record.get(c);
+if (columnC.array() != null) {
+System.arraycopy(record.get(c).array(), columnC.offset(), 
buf.array(), buf.offset() + pos, columnC.length());
+pos += columnC.length();
+} else {
+int maxLength = 
record.getInfo().getCodeSystem().maxCodeLength(c);
+Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + 
pos + maxLength, defaultValue);
+pos += maxLength;
+}
+}
+buf.setLength(pos);
+}
+
 @Override
 public void encode(ByteArray bodyBytes, ByteArray outputBuf) {
 Preconditions.checkState(bodyBytes.length() == bodyLength);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index b676693..276f66e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -235,22 +235,7 @@ public class GTRecord implements Comparable {
 buf.setLength(pos);
 }
 
-/** write data to given buffer, like serialize, use defaultValue when 
required column is not set*/
-public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf, 
byte defaultValue) {
-int pos = 0;
-for (int i = 0; i < selectedCols.trueBitCount(); i++) {
-int c = selectedCols.trueBitAt(i);
-if (cols[c].array() != null) {
-System.arraycopy(cols[c].array(), cols[c].offset(), 
buf.array(), buf.offset() + pos, cols[c].length());
-pos += cols[c].length();
-} else {
-int maxLength = info.codeSystem.maxCodeLength(c);
-Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + 
pos + maxLength, defaultValue);
-pos += maxLength;
-}
-}
-buf.setLength(pos);
-}
+   
 
 /** write data to given buffer, like serialize */
 public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
--
diff

kylin git commit: minor refactoring

2016-06-19 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 11e20adbd -> 762992d0a


minor refactoring


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/762992d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/762992d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/762992d0

Branch: refs/heads/master
Commit: 762992d0ac981549f91587a73144e2d554662425
Parents: 11e20ad
Author: Hongbin Ma 
Authored: Mon Jun 20 11:43:54 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Jun 20 11:44:06 2016 +0800

--
 .../apache/kylin/common/util/SparkEntry.java| 42 +++
 .../apache/kylin/engine/spark/SparkCubing.java  |  1 +
 .../apache/kylin/engine/spark/SparkEntry.java   | 43 
 .../kylin/engine/spark/SparkExecutable.java |  2 +-
 4 files changed, 44 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/762992d0/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java 
b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
new file mode 100644
index 000..fd324a2
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ */
+public final class SparkEntry {
+
+public static void main(String[] args) throws Exception {
+System.out.println("SparkEntry args:" + StringUtils.join(args, " "));
+Preconditions.checkArgument(args.length >= 2, "-className is 
required");
+Preconditions.checkArgument(args[0].equals("-className"), "-className 
is required");
+final String className = args[1];
+final Object o = Class. 
forName(className).newInstance();
+Preconditions.checkArgument(o instanceof AbstractApplication, 
className + " is not a subClass of AbstractSparkApplication");
+String[] appArgs = new String[args.length - 2];
+for (int i = 2; i < args.length; i++) {
+appArgs[i - 2] = args[i];
+}
+AbstractApplication abstractApplication = (AbstractApplication) o;
+abstractApplication.execute(appArgs);
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/762992d0/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index c56892b..2c7a8fa 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -610,6 +610,7 @@ public class SparkCubing extends AbstractApplication {
 
 }
 });
+
 final Map samplingResult = 
sampling(rowJavaRDD, cubeName);
 final byte[][] splitKeys = createHTable(cubeName, segmentId, 
samplingResult);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/762992d0/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkEntry.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkEntry.java 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkEntry.java
deleted file mode 100644
index 35d223a..000
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkEntry.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with

kylin git commit: KYLIN-1781 fix config encoding error

2016-06-15 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 7a05309bf -> a179136be


KYLIN-1781 fix config encoding error

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a179136b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a179136b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a179136b

Branch: refs/heads/master
Commit: a179136beabcad7a2cbb766d448d9acf88194245
Parents: 7a05309
Author: zhaotianshuo 
Authored: Mon Jun 13 13:33:55 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Jun 16 13:45:47 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfig.java | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a179136b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 580edd8..b4b3c1b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.common;
 
+import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -26,6 +27,7 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.util.Enumeration;
@@ -197,9 +199,9 @@ public class KylinConfig extends KylinConfigBase {
 fis = new FileInputStream(propFile);
 fis2 = new FileInputStream(overrideFile);
 Properties conf = new Properties();
-conf.load(fis);
+conf.load(new BufferedReader(new InputStreamReader(fis, 
"UTF-8")));
 Properties override = new Properties();
-override.load(fis2);
+override.load(new BufferedReader(new InputStreamReader(fis2, 
"UTF-8")));
 for (Map.Entry entries : override.entrySet()) {
 conf.setProperty(entries.getKey().toString(), 
entries.getValue().toString());
 }

kylin git commit: update doc for deploy cluster

2016-05-17 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/document 4841ea0d3 -> c27d818b5


update doc for deploy cluster


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c27d818b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c27d818b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c27d818b

Branch: refs/heads/document
Commit: c27d818b510bbe03e089a11dbfea202ac0f6ff57
Parents: 4841ea0
Author: Hongbin Ma 
Authored: Wed May 18 11:31:31 2016 +0800
Committer: Hongbin Ma 
Committed: Wed May 18 11:31:31 2016 +0800

--
 website/_docs15/install/kylin_cluster.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c27d818b/website/_docs15/install/kylin_cluster.md
--
diff --git a/website/_docs15/install/kylin_cluster.md 
b/website/_docs15/install/kylin_cluster.md
index 0da35fe..97947be 100644
--- a/website/_docs15/install/kylin_cluster.md
+++ b/website/_docs15/install/kylin_cluster.md
@@ -18,15 +18,15 @@ A typical scenario is depicted in the following chart:
 
 ### Setting up Multiple Kylin REST servers
 
-If you are running Kylin in a cluster or you have multiple Kylin REST server 
instances, please make sure you have the following property correctly 
configured in ${KYLIN_HOME}/conf/kylin.properties
+If you are running Kylin in a cluster where you have multiple Kylin REST 
server instances, please make sure you have the following property correctly 
configured in ${KYLIN_HOME}/conf/kylin.properties for EVERY server instance.
 
 1. kylin.rest.servers 
List of web servers in use, this enables one web server instance to 
sync up with other servers. For example: 
kylin.rest.servers=sandbox1:7070,sandbox2:7070
   
 2. kylin.server.mode
-   Make sure there is only one instance whose "kylin.server.mode" is set 
to "all" if there are multiple instances.
+   Make sure there is only one instance whose "kylin.server.mode" is set 
to "all"(or "job"), others should be "query"

 ## Setup load balancer 
 
 To enable Kylin high availability, you need setup a load balancer in front of 
these servers, let it routing the incoming requests to the cluster. Client 
sides send all requests to the load balancer, instead of talk with a specific 
instance. 
-   
\ No newline at end of file
+

kylin git commit: minor changes

2016-05-17 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 06e1ea296 -> 4a6aed92e


minor changes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a6aed92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a6aed92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a6aed92

Branch: refs/heads/master
Commit: 4a6aed92ea9ad1d155037dda756a19e55d50dec2
Parents: 06e1ea2
Author: Hongbin Ma 
Authored: Tue May 17 18:45:20 2016 +0800
Committer: Hongbin Ma 
Committed: Tue May 17 18:45:20 2016 +0800

--
 .../kylin/common/util/NumberIterators.java  | 50 
 .../kylin/common/util/ValueIterators.java   | 50 
 .../apache/kylin/rest/response/SQLResponse.java |  4 ++
 3 files changed, 54 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4a6aed92/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java 
b/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java
deleted file mode 100644
index 4ce317d..000
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- * http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.Iterator;
-import java.util.Set;
-
-import com.google.common.collect.Sets;
-
-public class NumberIterators {
-
-public static  long sum(Iterator iterator) {
-long ret = 0;
-while (iterator.hasNext()) {
-Number element = iterator.next();
-ret += element.longValue();
-}
-return ret;
-}
-
-public static  long checkSame(Iterator iterator) {
-Set values = Sets.newHashSet();
-while (iterator.hasNext()) {
-Number element = iterator.next();
-values.add(element.longValue());
-}
-
-if (values.size() > 1) {
-throw new IllegalStateException("more than one distinct values 
exist in the collection:" + values);
-}
-
-return values.iterator().next();
-}
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/4a6aed92/core-common/src/main/java/org/apache/kylin/common/util/ValueIterators.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/ValueIterators.java 
b/core-common/src/main/java/org/apache/kylin/common/util/ValueIterators.java
new file mode 100644
index 000..9ed85be
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ValueIterators.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.util.Iterator;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+
+public class ValueIterators {
+
+public static  long sum(Iterator iterator) {
+long ret = 0;
+while (iterator.hasNext()) {
+Number element = iterator.next();
+ret += element.longValue();
+}
+

kylin git commit: minor changes

2016-05-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 407fbd0d4 -> 58f3af3b3


minor changes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58f3af3b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58f3af3b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58f3af3b

Branch: refs/heads/master
Commit: 58f3af3b33d7d581811e1854a4c14e4f2c6edaf8
Parents: 407fbd0
Author: Hongbin Ma 
Authored: Fri May 13 09:55:15 2016 +0800
Committer: Hongbin Ma 
Committed: Fri May 13 13:54:10 2016 +0800

--
 .../apache/kylin/common/util/NumberIterators.java   | 16 
 .../apache/kylin/rest/model/SelectedColumnMeta.java |  4 +++-
 .../org/apache/kylin/rest/response/SQLResponse.java |  8 
 3 files changed, 27 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58f3af3b/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java 
b/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java
index 5bfbac7..4ce317d 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/NumberIterators.java
@@ -19,6 +19,9 @@
 package org.apache.kylin.common.util;
 
 import java.util.Iterator;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
 
 public class NumberIterators {
 
@@ -31,4 +34,17 @@ public class NumberIterators {
 return ret;
 }
 
+public static  long checkSame(Iterator iterator) {
+Set values = Sets.newHashSet();
+while (iterator.hasNext()) {
+Number element = iterator.next();
+values.add(element.longValue());
+}
+
+if (values.size() > 1) {
+throw new IllegalStateException("more than one distinct values 
exist in the collection:" + values);
+}
+
+return values.iterator().next();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/58f3af3b/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java 
b/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java
index 00b01db..fc8d723 100644
--- a/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java
+++ b/server/src/main/java/org/apache/kylin/rest/model/SelectedColumnMeta.java
@@ -18,9 +18,11 @@
 
 package org.apache.kylin.rest.model;
 
+import java.io.Serializable;
+
 /**
  */
-public class SelectedColumnMeta {
+public class SelectedColumnMeta implements Serializable{
 public SelectedColumnMeta(boolean isAutoIncrement, boolean 
isCaseSensitive, boolean isSearchable, boolean isCurrency, int isNullalbe, 
boolean isSigned, int displaySize, String label, String name, String 
schemaName, String catelogName, String tableName, int precision, int scale, int 
columnType, String columnTypeName, boolean isReadOnly, boolean isWritable, 
boolean isDefinitelyWritable) {
 super();
 this.isAutoIncrement = isAutoIncrement;

http://git-wip-us.apache.org/repos/asf/kylin/blob/58f3af3b/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java 
b/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index 75b6ce8..353dd7e 100644
--- a/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -113,10 +113,18 @@ public class SQLResponse implements Serializable {
 return isException;
 }
 
+public void setIsException(boolean v) {
+isException = v;
+}
+
 public String getExceptionMessage() {
 return exceptionMessage;
 }
 
+public void setExceptionMessage(String msg) {
+exceptionMessage = msg;
+}
+
 public long getDuration() {
 return duration;
 }

kylin git commit: make SQLRequest serializable

2016-05-10 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 2085686ac -> f4e171b06


make SQLRequest serializable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f4e171b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f4e171b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f4e171b0

Branch: refs/heads/master
Commit: f4e171b06a699f40bf2db5ec6cdc937c38cf43b1
Parents: 2085686
Author: Hongbin Ma 
Authored: Tue May 10 18:42:20 2016 +0800
Committer: Hongbin Ma 
Committed: Tue May 10 18:43:08 2016 +0800

--
 .../src/main/java/org/apache/kylin/rest/request/SQLRequest.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f4e171b0/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
--
diff --git a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java 
b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index 8b574a8..96f5faa 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -18,9 +18,11 @@
 
 package org.apache.kylin.rest.request;
 
+import java.io.Serializable;
 import java.util.Map;
 
-public class SQLRequest {
+public class SQLRequest implements Serializable {
+protected static final long serialVersionUID = 1L;
 
 private String sql;
 private String project;

kylin git commit: refactor

2016-05-09 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 656c645f3 -> 2fc706ca1


refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2fc706ca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2fc706ca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2fc706ca

Branch: refs/heads/master
Commit: 2fc706ca132dfad43b2f68c41757ddfbadc595b0
Parents: 656c645
Author: Hongbin Ma 
Authored: Mon May 9 18:38:13 2016 +0800
Committer: Hongbin Ma 
Committed: Mon May 9 18:40:06 2016 +0800

--
 build/conf/kylin.properties |  3 +-
 .../apache/kylin/common/KylinConfigBase.java| 14 +++-
 .../kylin/common/debug/BackdoorToggles.java | 24 ++
 .../kylin/common/restclient/RestClient.java | 16 ++--
 .../common/util/LoggableCachedThreadPool.java   | 57 ++
 .../kylin/common/util/NumberIterators.java  | 34 
 .../apache/kylin/gridtable/ScannerWorker.java   |  3 +-
 .../apache/kylin/storage/cache/EhcacheTest.java | 49 
 .../apache/kylin/rest/response/SQLResponse.java | 26 +++
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 82 +++-
 10 files changed, 213 insertions(+), 95 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2fc706ca/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 80ae004..9a2fd81 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -168,4 +168,5 @@ kylin.web.hive.limit=20
 # E.g, for cube whose capacity be marked as "SMALL", split region per 5GB by 
default
 kylin.hbase.region.cut.small=5
 kylin.hbase.region.cut.medium=10
-kylin.hbase.region.cut.large=50
\ No newline at end of file
+kylin.hbase.region.cut.large=50
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/2fc706ca/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 850ea17..3db97b3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -361,8 +361,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptionalStringArray("kylin.rest.servers", new String[0]);
 }
 
-public String[] getAdminDls() {
-return getOptionalStringArray("kylin.job.admin.dls", null);
+public int getWorkersPerServer() {
+//for sequence sql use
+return Integer.parseInt(getOptional("kylin.rest.workers.per.server", 
"1"));
+}
+
+public String getAdminDls() {
+return getOptional("kylin.job.admin.dls", null);
 }
 
 public long getJobStepTimeout() {
@@ -498,6 +503,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", 
"10485760"));
 }
 
+public String getDefaultIGTStorage() {
+return getOptional("kylin.query.storage.default.gtstorage", 
"org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC");
+}
+
 public int getHBaseScanCacheRows() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
@@ -632,4 +641,5 @@ abstract public class KylinConfigBase implements 
Serializable {
 public int getCubeStatsHLLPrecision() {
 return 
Integer.parseInt(getOptional("kylin.job.cubing.inmem.sampling.hll.precision", 
"14"));
 }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2fc706ca/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 0feff2d..f88781a 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -20,6 +20,9 @@ package org.apache.kylin.common.debug;
 
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.Pair;
+
 /**
  */
 public class BackdoorToggles {
@@ -62,6 +65,16 @@ public class BackdoorToggles {
 return Integer.valueOf(v);
 }
 
+public static Pair getShardAssignment() {
+String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
+if (v ==

kylin git commit: minor

2016-04-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/mhb_test_1 5fd272892 -> df287ba08


minor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df287ba0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df287ba0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df287ba0

Branch: refs/heads/mhb_test_1
Commit: df287ba08b2a21eb39ff2a359515f580d5fd4571
Parents: 5fd2728
Author: Hongbin Ma 
Authored: Wed Apr 27 13:22:57 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 27 13:22:57 2016 +0800

--
 .../apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/df287ba0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 75cc977..f24aeb7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -361,7 +361,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 logger.info("Checking 
BackdoorToggles.getStorageResultDump()");
 if (storageResultDumpPath != null) {
 try {
-FileUtils.writeByteArrayToFile(new 
File(BackdoorToggles.getStorageResultDump() + "/" + order), compressed);
+FileUtils.writeByteArrayToFile(new 
File(storageResultDumpPath + "/" + order), compressed);
 logger.info("write to file for 
part " + order + " finished");
 } catch (Exception e) {
 logger.error("failed to write for 
part " + order, e);

kylin git commit: bug fix

2016-04-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/mhb_test_1 7c154921e -> 5fd272892


bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5fd27289
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5fd27289
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5fd27289

Branch: refs/heads/mhb_test_1
Commit: 5fd2728929ff29a90b062fe6a97049178149e86f
Parents: 7c15492
Author: Hongbin Ma 
Authored: Wed Apr 27 13:11:01 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 27 13:11:01 2016 +0800

--
 .../apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5fd27289/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index f554dac..75cc977 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -322,6 +322,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 final AtomicInteger totalScannedCount = new AtomicInteger(0);
 final ExpectedSizeIterator epResultItr = new 
ExpectedSizeIterator(scanRequests.size() * shardNum);
 final AtomicInteger atomicInteger = new AtomicInteger(0);
+final String storageResultDumpPath = 
BackdoorToggles.getStorageResultDump();//usually it's null
 
 for (final Pair epRange : 
getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
 executorService.submit(new Runnable() {
@@ -358,7 +359,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 int order = 
atomicInteger.incrementAndGet();
 byte[] compressed = 
HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows());
 logger.info("Checking 
BackdoorToggles.getStorageResultDump()");
-if (BackdoorToggles.getStorageResultDump() 
!= null) {
+if (storageResultDumpPath != null) {
 try {
 FileUtils.writeByteArrayToFile(new 
File(BackdoorToggles.getStorageResultDump() + "/" + order), compressed);
 logger.info("write to file for 
part " + order + " finished");

kylin git commit: KYLIN-1627 add backdoor toggle to dump binary cube storage response for further analysis

2016-04-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/mhb_test_1 [created] 7405e48fc


KYLIN-1627 add backdoor toggle to dump binary cube storage response for further 
analysis


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7405e48f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7405e48f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7405e48f

Branch: refs/heads/mhb_test_1
Commit: 7405e48fcee98e29ffa319e6744eae9825e9ef61
Parents: aebef62
Author: Hongbin Ma 
Authored: Wed Apr 27 10:58:51 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 27 10:59:08 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java | 14 ++
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 20 +---
 2 files changed, 31 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7405e48f/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 0feff2d..4a4a2f9 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -42,6 +42,10 @@ public class BackdoorToggles {
 return getString(DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL);
 }
 
+public static String getStorageResultDump() {
+return getString(DEBUG_TOGGLE_STORAGE_RESULT_DUMP);
+}
+
 public static boolean getDisableCache() {
 return getBoolean(DEBUG_TOGGLE_DISABLE_QUERY_CACHE);
 }
@@ -153,4 +157,14 @@ public class BackdoorToggles {
  }
  */
 public final static String DEBUG_TOGGLE_QUERY_TIMEOUT = 
"DEBUG_TOGGLE_QUERY_TIMEOUT";
+
+/**
+ * set DEBUG_TOGGLE_STORAGE_RESULT_DUMP="path_to_dump" to dump the result 
from cube storage
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_STORAGE_RESULT_DUMP": "/kylin/data/"
+ }
+ */
+public final static String DEBUG_TOGGLE_STORAGE_RESULT_DUMP = 
"DEBUG_TOGGLE_STORAGE_RESULT_DUMP";
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7405e48f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 99a72bb..5d17764 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.storage.hbase.cube.v2;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
@@ -36,6 +37,7 @@ import java.util.zip.DataFormatException;
 
 import javax.annotation.Nullable;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HConnection;
@@ -125,11 +127,11 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 this.timeout = 
HadoopUtil.getCurrentConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 
HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
 this.timeout *= 
KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
-
+
 if (BackdoorToggles.getQueryTimeout() != -1) {
 this.timeout = BackdoorToggles.getQueryTimeout();
 }
-
+
 this.timeout *= 1.1;//allow for some delay 
 
 logger.info("Timeout for ExpectedSizeIterator is: " + 
this.timeout);
@@ -319,6 +321,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 final AtomicInteger totalScannedCount = new AtomicInteger(0);
 final ExpectedSizeIterator epResultItr = new 
ExpectedSizeIterator(scanRequests.size() * shardNum);
+final AtomicInteger atomicInteger = new AtomicInteger(0);
 
 for (final Pair epRange : 
getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
 executorService.submit(new Runnable() {
@@ -352,7 +355,17 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 abnormalFinish = true;
 } else {
 try {
-

[2/2] kylin git commit: minor improve

2016-04-26 Thread mahongbin

minor improve


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d30bc8d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d30bc8d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d30bc8d5

Branch: refs/heads/master
Commit: d30bc8d542557a6fe92e041c4282c08032c89641
Parents: d899a67
Author: Hongbin Ma 
Authored: Mon Apr 25 13:40:16 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 26 16:07:26 2016 +0800

--
 .../apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d30bc8d5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 525625b..cdacec7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -127,11 +127,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 this.timeout = 
HadoopUtil.getCurrentConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 
HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
 this.timeout *= 
KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
-this.timeout *= 1.1;//allow for some delay 
 
 if (BackdoorToggles.getQueryTimeout() != -1) {
 this.timeout = BackdoorToggles.getQueryTimeout();
 }
+
+this.timeout *= 1.1;//allow for some delay 
 
 logger.info("Timeout for ExpectedSizeIterator is: " + 
this.timeout);

kylin git commit: KYLIN-1621 set deflate to level 1

2016-04-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 0d9da9446 -> aebef6239


KYLIN-1621 set deflate to level 1


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aebef623
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aebef623
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aebef623

Branch: refs/heads/yang-m1
Commit: aebef6239b42c079060c30377130d906fb5839e2
Parents: 0d9da94
Author: Hongbin Ma 
Authored: Tue Apr 26 16:07:20 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 26 16:07:53 2016 +0800

--
 .../main/java/org/apache/kylin/common/util/CompressionUtils.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aebef623/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
index 58c10ff..2781cac 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
@@ -31,7 +31,7 @@ public class CompressionUtils {
 private static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(CompressionUtils.class);
 
 public static byte[] compress(byte[] data) throws IOException {
-Deflater deflater = new Deflater();
+Deflater deflater = new Deflater(1);
 deflater.setInput(data);
 
 ByteArrayOutputStream outputStream = new 
ByteArrayOutputStream(data.length);

[1/2] kylin git commit: KYLIN-1621 set deflate to level 1

2016-04-26 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master d899a6796 -> aebdd66fe


KYLIN-1621 set deflate to level 1


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aebdd66f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aebdd66f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aebdd66f

Branch: refs/heads/master
Commit: aebdd66fec3739783d247e253e3f2782aac25a6b
Parents: d30bc8d
Author: Hongbin Ma 
Authored: Tue Apr 26 16:07:20 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 26 16:07:26 2016 +0800

--
 .../main/java/org/apache/kylin/common/util/CompressionUtils.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aebdd66f/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java 
b/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
index 58c10ff..2781cac 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/CompressionUtils.java
@@ -31,7 +31,7 @@ public class CompressionUtils {
 private static final org.slf4j.Logger logger = 
LoggerFactory.getLogger(CompressionUtils.class);
 
 public static byte[] compress(byte[] data) throws IOException {
-Deflater deflater = new Deflater();
+Deflater deflater = new Deflater(1);
 deflater.setInput(data);
 
 ByteArrayOutputStream outputStream = new 
ByteArrayOutputStream(data.length);

kylin git commit: enable query level timeout backdoor toggle and correct ExpectedSizeIterator wait timeout

2016-04-24 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master e1c2b4f53 -> 85d9dff37


enable query level timeout backdoor toggle and correct ExpectedSizeIterator 
wait timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85d9dff3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85d9dff3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85d9dff3

Branch: refs/heads/master
Commit: 85d9dff37f5363fa3a1be6ee64324b88610f0fc4
Parents: e1c2b4f
Author: Hongbin Ma 
Authored: Mon Apr 25 13:35:56 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 25 13:35:56 2016 +0800

--
 .../kylin/common/debug/BackdoorToggles.java | 18 ++
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 20 +---
 2 files changed, 35 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85d9dff3/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index f3745d7..0feff2d 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -54,6 +54,14 @@ public class BackdoorToggles {
 return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
 }
 
+public static int getQueryTimeout() {
+String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT);
+if (v == null)
+return -1;
+else
+return Integer.valueOf(v);
+}
+
 private static String getString(String key) {
 Map toggles = _backdoorToggles.get();
 if (toggles == null) {
@@ -135,4 +143,14 @@ public class BackdoorToggles {
  }
  */
 public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = 
"DEBUG_TOGGLE_LOCAL_COPROCESSOR";
+
+/**
+ * set DEBUG_TOGGLE_QUERY_TIMEOUT="timeout_millis" to overwrite the global 
timeout settings
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_QUERY_TIMEOUT": "12"
+ }
+ */
+public final static String DEBUG_TOGGLE_QUERY_TIMEOUT = 
"DEBUG_TOGGLE_QUERY_TIMEOUT";
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/85d9dff3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 88c17ae..525625b 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -114,19 +114,28 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 static class ExpectedSizeIterator implements Iterator {
 
+BlockingQueue queue;
+
 int expectedSize;
 int current = 0;
-BlockingQueue queue;
 long timeout;
+long timeoutTS;
 
 public ExpectedSizeIterator(int expectedSize) {
 this.expectedSize = expectedSize;
 this.queue = new ArrayBlockingQueue(expectedSize);
+
 this.timeout = 
HadoopUtil.getCurrentConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 
HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
 this.timeout *= 
KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
-
 this.timeout *= 1.1;//allow for some delay 
+
+if (BackdoorToggles.getQueryTimeout() != -1) {
+this.timeout = BackdoorToggles.getQueryTimeout();
+}
+
 logger.info("Timeout for ExpectedSizeIterator is: " + 
this.timeout);
+
+this.timeoutTS = System.currentTimeMillis() + this.timeout;
 }
 
 @Override
@@ -141,7 +150,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 }
 try {
 current++;
-byte[] ret = queue.poll(timeout, TimeUnit.MILLISECONDS);
+long tsRemaining = this.timeoutTS - System.currentTimeMillis();
+if (tsRemaining < 0) {
+throw new RuntimeException("Timeout visiting cube!");
+}
+
+byte[] ret = queue.poll(tsRemaining, TimeUnit.MILLISECONDS);
 if (ret == null) {
 throw new

kylin git commit: refine coprocessor suicide

2016-04-21 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 1baa91473 -> 638f7b3c8


refine coprocessor suicide


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/638f7b3c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/638f7b3c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/638f7b3c

Branch: refs/heads/yang-m1
Commit: 638f7b3c8b59b9a70a281352fee243dc424daa12
Parents: 1baa914
Author: Hongbin Ma 
Authored: Thu Apr 21 16:05:31 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 21 16:05:31 2016 +0800

--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  2 ++
 .../coprocessor/endpoint/CubeVisitService.java  | 35 
 2 files changed, 30 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/638f7b3c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index cf39641..053d99e 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -122,6 +122,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 this.queue = new ArrayBlockingQueue(expectedSize);
 this.timeout = 
HadoopUtil.getCurrentConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 
HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
 this.timeout *= 
KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
+
+this.timeout *= 1.1;//allow for some delay 
 logger.info("Timeout for ExpectedSizeIterator is " + this.timeout);
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/638f7b3c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index f6476f8..6058768 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -106,13 +106,20 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 
 @Override
 public boolean hasNext() {
-if (counter++ % 1000 == 1) {
+if (counter % 1000 == 1) {
 if (System.currentTimeMillis() - startTime > timeout) {
 normalComplete.setValue(false);
+logger.error("scanner aborted because timeout");
 return false;
 }
 }
 
+if (counter % 10 == 1) {
+logger.info("Scanned " + counter + " rows.");
+}
+
+counter++;
+
 return !nextOne.isEmpty();
 }
 
@@ -218,10 +225,10 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 appendProfileInfo(sb, "scanned " + counter);
 }
 
-final MutableBoolean normalComplete = new MutableBoolean(true);
-final long startTime = 
this.serviceStartTime;//request.getStartTime();
-final long timeout = (long) (request.getTimeout() * 0.95);
-InnerScannerAsIterator cellListIterator = new 
InnerScannerAsIterator(innerScanner, normalComplete, startTime, timeout);
+final MutableBoolean scanNormalComplete = new MutableBoolean(true);
+final long startTime = this.serviceStartTime;
+final long timeout = request.getTimeout();
+InnerScannerAsIterator cellListIterator = new 
InnerScannerAsIterator(innerScanner, scanNormalComplete, startTime, timeout);
 
 if (behavior.ordinal() < 
CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
 scanReq.setAggrCacheGB(0); // disable mem check if so told
@@ -239,6 +246,20 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 ByteArrayOutputStream outputStream = new 
ByteArrayOutputStream(RowConstants.ROWVALUE_BUFFER_SIZE);//ByteArrayOutputStream
 will auto grow
 int finalRowCount = 0;
 for (GTRecord

kylin git commit: KYLIN-1542 change all hfile to 777 before bulk loading

2016-04-20 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master f76190fe9 -> 618a66073


KYLIN-1542 change all hfile to 777 before bulk loading


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/618a6607
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/618a6607
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/618a6607

Branch: refs/heads/master
Commit: 618a6607348479e46dced04983c438d98366fd31
Parents: f76190f
Author: Hongbin Ma 
Authored: Thu Apr 21 11:58:13 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 21 11:58:22 2016 +0800

--
 .../apache/kylin/engine/spark/SparkCubing.java  | 199 ++-
 .../kylin/storage/hbase/ii/IIBulkLoadJob.java   |  21 +-
 .../kylin/storage/hbase/steps/BulkLoadJob.java  |  34 +---
 3 files changed, 124 insertions(+), 130 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/618a6607/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index fc7f4b1..ef35067 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -17,12 +17,26 @@
 */
 package org.apache.kylin.engine.spark;
 
-import com.google.common.base.*;
-import com.google.common.collect.*;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
-import com.google.common.primitives.UnsignedBytes;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import javax.annotation.Nullable;
 
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
@@ -30,8 +44,8 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -40,10 +54,10 @@ import 
org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
@@ -54,18 +68,22 @@ import 
org.apache.kylin.cube.inmemcubing.AbstractInMemCubeBuilder;
 import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
 import org.apache.kylin.cube.kv.CubeDimEncMap;
 import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.*;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.cube.model.RowKeyDesc;
 import org.apache.kylin.cube.util.CubingUtils;
-import org.apache.kylin.dict.*;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
 import org.apache.kylin.dimension.Dictionary;
+import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.engine.mr.common.CubeStatsReader;
 import org.apache.kylin.engine.spark.cube.BufferedCuboidWriter;
 import org.apache.kylin.engine.spark.cube.DefaultTupleConverter;
 import org.apache.kylin.engine.spark.util.IteratorUtils;
-import org.apache.kylin.common.util.AbstractApplication;
-import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.measure.MeasureAggregators;
 import org.apache.kylin.measure.MeasureCodec;
+import

kylin git commit: KYLIN-1595 add hll to with slr left cube

2016-04-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 8ee67a9cd -> 01ae37d5d


KYLIN-1595 add hll to with slr left cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01ae37d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01ae37d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01ae37d5

Branch: refs/heads/master
Commit: 01ae37d5dab5dcc837a1286344f7f7d85350d7c5
Parents: 8ee67a9
Author: Hongbin Ma 
Authored: Tue Apr 19 11:59:39 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 19 11:59:39 2016 +0800

--
 ...test_kylin_cube_with_slr_left_join_desc.json | 407 ---
 .../apache/kylin/query/ITMassInQueryTest.java   |   4 +-
 .../query/sql_massin_distinct/query01.sql   |   2 +-
 3 files changed, 255 insertions(+), 158 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/01ae37d5/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index f0fe9c0..608b9de 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -1,171 +1,266 @@
 {
-  "uuid" : "a905-1fc6-4f67-985c-38fa5aeafd92",
- 
-  "name" : "test_kylin_cube_with_slr_left_join_desc",
-  "description" : null,
-  "dimensions" : [ {
-"name" : "CAL_DT",
-"table" : "EDW.TEST_CAL_DT",
-"column" : "{FK}",
-"derived" : [ "WEEK_BEG_DT" ]
-  }, {
-"name" : "CATEGORY",
-"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
-"column" : "META_CATEG_NAME",
-"derived" : null
-  }, {
-"name" : "CATEGORY",
-"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
-"column" : "CATEG_LVL2_NAME",
-"derived" : null
-  }, {
-"name" : "CATEGORY",
-"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
-"column" : "CATEG_LVL3_NAME",
-"derived" : null
-  }, {
-"name" : "CATEGORY_DERIVED",
-"table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
-"column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER" ]
-  }, {
-"name" : "LSTG_FORMAT_NAME",
-"table" : "DEFAULT.TEST_KYLIN_FACT",
-"column" : "LSTG_FORMAT_NAME",
-"derived" : null
-  }, {
-"name" : "SITE_ID",
-"table" : "EDW.TEST_SITES",
-"column" : "{FK}",
-"derived" : [ "SITE_NAME", "CRE_USER" ]
-  }, {
-"name" : "SELLER_TYPE_CD",
-"table" : "EDW.TEST_SELLER_TYPE_DIM",
-"column" : "{FK}",
-"derived" : [ "SELLER_TYPE_DESC" ]
-  }, {
-"name" : "SELLER_ID",
-"table" : "DEFAULT.TEST_KYLIN_FACT",
-"column" : "SELLER_ID",
-"derived" : null
-  } ],
-  "measures" : [ {
-"name" : "GMV_SUM",
-"function" : {
-  "expression" : "SUM",
-  "parameter" : {
-"type" : "column",
-"value" : "PRICE",
-"next_parameter" : null
+  "uuid": "a905-1fc6-4f67-985c-38fa5aeafd92",
+  "name": "test_kylin_cube_with_slr_left_join_desc",
+  "description": null,
+  "dimensions": [
+{
+  "name": "CAL_DT",
+  "table": "EDW.TEST_CAL_DT",
+  "column": "{FK}",
+  "derived": [
+"WEEK_BEG_DT"
+  ]
+},
+{
+  "name": "CATEGORY",
+  "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+  "column": "META_CATEG_NAME",
+  "derived": null
+},
+{
+  "name": "CATEGORY",
+  "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+  "column": "CATEG_LVL2_NAME",
+  "derived": null
+},
+{
+  "name": "CATEGORY",
+  "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+  "column": "CATEG_LVL3_NAME",
+  "derived": null
+},
+{
+  "name": "CATEGORY_DERIVED",
+  "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+  "column": "{FK}",
+  "derived": [
+"USER_DEFINED_FIELD1",
+"USER_DEFINED_FIELD3",
+"UPD_DATE",
+"UPD_USER"
+  ]
+},
+{
+  "name": "LSTG_FORMAT_NAME",
+  "table": "DEFAULT.TEST_KYLIN_FACT",
+  "column": "LSTG_FORMAT_NAME",
+  "derived": null
+},
+{
+  "name": "SITE_ID",
+  "table": "EDW.TEST_SITES",
+  "column": "{FK}",
+  "derived": [
+"SITE_NAME",
+"CRE_USER"
+  ]
+},
+{
+  "name": "SELLER_TYPE_CD",
+  "table": "EDW.TEST_SELLER_TYPE_DIM",
+  "column": "{FK}",
+  "derived": [
+"SELLER_TYPE_DESC"
+  ]
+},
+{
+  "name": "SELLER_ID",
+  "table": "DEFAULT.TEST_KYLIN_FACT",
+  "column": "SELLER_ID",
+

kylin git commit: KYLIN-1596 UT fail, due to String encoding CharsetEncoder mismatch

2016-04-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master ef36d6cb4 -> 8ee67a9cd


KYLIN-1596 UT fail, due to String encoding CharsetEncoder mismatch

Signed-off-by: wangxianbin1987 
Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ee67a9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ee67a9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ee67a9c

Branch: refs/heads/master
Commit: 8ee67a9cde465f806a8520009b50b73074564440
Parents: ef36d6c
Author: wangxianbin1987 
Authored: Tue Apr 19 10:04:02 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 19 10:24:18 2016 +0800

--
 .../apache/kylin/invertedindex/InvertedIndexLocalTest.java| 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8ee67a9c/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
--
diff --git 
a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
 
b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
index 4b74fdd..5aff1b0 100644
--- 
a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
+++ 
b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -171,7 +172,11 @@ public class InvertedIndexLocalTest extends 
LocalFileMetadataTestCase {
 @Nullable
 @Override
 public byte[] apply(String input) {
-return input.getBytes();
+try {
+return input.getBytes("UTF-8");
+} catch (UnsupportedEncodingException e) {
+throw new RuntimeException(e);
+}
 }
 })));
 }

kylin git commit: KYLIN-1596 UT fail, due to String encoding CharsetEncoder mismatch

2016-04-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 aef842442 -> 6ff4b940e


KYLIN-1596 UT fail, due to String encoding CharsetEncoder mismatch

Signed-off-by: wangxianbin1987 
Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ff4b940
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ff4b940
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ff4b940

Branch: refs/heads/yang-m1
Commit: 6ff4b940ef7c723cfb7f9c2508e8759992fdea2c
Parents: aef8424
Author: wangxianbin1987 
Authored: Tue Apr 19 10:04:02 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 19 10:23:46 2016 +0800

--
 .../apache/kylin/invertedindex/InvertedIndexLocalTest.java| 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6ff4b940/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
--
diff --git 
a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
 
b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
index 4b74fdd..5aff1b0 100644
--- 
a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
+++ 
b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -171,7 +172,11 @@ public class InvertedIndexLocalTest extends 
LocalFileMetadataTestCase {
 @Nullable
 @Override
 public byte[] apply(String input) {
-return input.getBytes();
+try {
+return input.getBytes("UTF-8");
+} catch (UnsupportedEncodingException e) {
+throw new RuntimeException(e);
+}
 }
 })));
 }

kylin git commit: KYLIN-1595 add mass distinct count query

2016-04-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 59477501c -> 205471998


KYLIN-1595 add mass distinct count query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20547199
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20547199
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20547199

Branch: refs/heads/master
Commit: 205471998d551bf5bc1988e0da73f5490f0a622c
Parents: 5947750
Author: Hongbin Ma 
Authored: Mon Apr 18 22:24:44 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 18 22:24:52 2016 +0800

--
 .../impl/threadpool/DefaultSchedulerTest.java   |  2 --
 .../apache/kylin/query/ITMassInQueryTest.java   | 27 -
 .../query/sql_massin_distinct/query01.sql   | 31 
 3 files changed, 57 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/20547199/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index 6acbbe5..df521f9 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -37,12 +37,10 @@ import org.apache.kylin.job.SucceedTestExecutable;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 import org.apache.kylin.job.execution.ExecutableState;
 import org.junit.Assert;
-import org.junit.Ignore;
 import org.junit.Test;
 
 /**
  */
-@Ignore
 public class DefaultSchedulerTest extends BaseSchedulerTest {
 
 @Test

http://git-wip-us.apache.org/repos/asf/kylin/blob/20547199/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index a92e298..5f2db43 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -63,7 +63,7 @@ public class ITMassInQueryTest extends KylinTestBase {
 public void setup() throws Exception {
 
 ITKylinQueryTest.clean();
-ITKylinQueryTest.joinType = "inner";
+ITKylinQueryTest.joinType = "left";
 ITKylinQueryTest.setupAll();
 
 Configuration hconf = HadoopUtil.getCurrentConfiguration();
@@ -96,6 +96,31 @@ public class ITMassInQueryTest extends KylinTestBase {
 compare("src/test/resources/query/sql_massin", null, true);
 }
 
+@Test
+public void testMassInWithDistinctCount() throws Exception {
+run("src/test/resources/query/sql_massin_distinct", null, true);
+}
+
+protected void run(String queryFolder, String[] exclusiveQuerys, boolean 
needSort) throws Exception {
+printInfo("-- test folder: " + queryFolder);
+Set exclusiveSet = buildExclusiveSet(exclusiveQuerys);
+
+List sqlFiles = getFilesFromFolder(new File(queryFolder), 
".sql");
+for (File sqlFile : sqlFiles) {
+String queryName = StringUtils.split(sqlFile.getName(), '.')[0];
+if (exclusiveSet.contains(queryName)) {
+continue;
+}
+String sql = getTextFromFile(sqlFile);
+
+// execute Kylin
+printInfo("Query Result from Kylin - " + queryName + "  (" + 
queryFolder + ")");
+IDatabaseConnection kylinConn = new 
DatabaseConnection(cubeConnection);
+ITable kylinTable = executeQuery(kylinConn, queryName, sql, 
needSort);
+
+}
+}
+
 protected void compare(String queryFolder, String[] exclusiveQuerys, 
boolean needSort) throws Exception {
 printInfo("-- test folder: " + queryFolder);
 Set exclusiveSet = buildExclusiveSet(exclusiveQuerys);

http://git-wip-us.apache.org/repos/asf/kylin/blob/20547199/kylin-it/src/test/resources/query/sql_massin_distinct/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_massin_distinct/query01.sql 
b/kylin-it/src/test/resources/query/sql_massin_distinct/query01.sql
new file mode 100644
index 000..70afa87
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_massin_distinct/query01.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+--

kylin git commit: KYLIN-1595 columns in metric could also be in filter/groupby

2016-04-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master f0473fcaf -> 78ff89470


KYLIN-1595 columns in metric could also be in filter/groupby


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78ff8947
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78ff8947
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78ff8947

Branch: refs/heads/master
Commit: 78ff89470a9f774af3a6d9d189393a67f823d483
Parents: f0473fc
Author: Hongbin Ma 
Authored: Mon Apr 18 18:27:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 18 18:27:09 2016 +0800

--
 .../localmeta/cube_desc/test_kylin_cube_with_slr_desc.json | 2 +-
 .../org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/78ff8947/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
index 417eb5c..7db40ec 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json
@@ -158,7 +158,7 @@
 "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", 
"leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name", 
"seller_id", "slr_segment_cd" ],
 "select_rule" : {
   "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", 
"CATEG_LVL3_NAME" ] ],
-  "mandatory_dims" : [ "seller_id" ],
+  "mandatory_dims" : [],
   "joint_dims" : [ [ "lstg_format_name", "lstg_site_id", "slr_segment_cd" 
] ]
 }
   } ],

http://git-wip-us.apache.org/repos/asf/kylin/blob/78ff8947/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index e0e6d83..b765798 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -136,7 +136,7 @@ public class CubeStorageQuery implements IStorageQuery {
 
 for (TblColRef column : sqlDigest.allColumns) {
 // skip measure columns
-if (sqlDigest.metricColumns.contains(column)) {
+if (sqlDigest.metricColumns.contains(column) && 
!(sqlDigest.groupbyColumns.contains(column) || 
sqlDigest.filterColumns.contains(column))) {
 continue;
 }

[3/3] kylin git commit: KYLIN-1591 enabling different block size setting for small column families

2016-04-18 Thread mahongbin

KYLIN-1591 enabling different block size setting for small column families


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3aad1fc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3aad1fc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3aad1fc6

Branch: refs/heads/yang-m1
Commit: 3aad1fc60cc0278833d4b5e0e5f2db02788cd3ba
Parents: ad09341
Author: Hongbin Ma 
Authored: Mon Apr 18 15:40:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 18 15:51:31 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  | 10 +++---
 .../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 +
 .../java/org/apache/kylin/gridtable/GTScanRequest.java |  3 ++-
 .../kylin/storage/hbase/steps/CubeHTableUtil.java  |  9 ++---
 4 files changed, 28 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 94d8f62..23f0fa7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -481,7 +481,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getQueryScanFuzzyKeyMax() {
-return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
 }
 
 public int getHBaseKeyValueSize() {
@@ -546,8 +546,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.hbase.default.encoding", "FAST_DIFF");
 }
 
-public String getHbaseDefaultBlockSize() {
-return getOptional("kylin.hbase.default.block.size", "4194304");
+public int getHbaseDefaultBlockSize() {
+return Integer.valueOf(getOptional("kylin.hbase.default.block.size", 
"1048576"));
+}
+
+public int getHbaseSmallFamilyBlockSize() {
+return 
Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536"));
 }
 
 public boolean isHiveKeepFlatTable() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
index 5db2a9b..c5b2e19 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.model;
 
 import java.util.Arrays;
 
+import org.apache.kylin.metadata.model.MeasureDesc;
+
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc {
 return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + 
Arrays.toString(columns) + "]";
 }
 
+public boolean isMemoryHungry() {
+for (HBaseColumnDesc hBaseColumnDesc : columns) {
+for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) {
+if 
(measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
+return true;
+}
+}
+}
+return false;
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 97900c1..befa866 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -153,7 +153,8 @@ public class GTScanRequest {
 }
 
 if (!doAggr) {//Skip reading this section if you're not profiling! 
-int scanned = lookAndForget(result);
+int scanned = result.getScannedRowCount();
+lookAndForget(result);
 return new EmptyGTScanner(scanned);
 }

[1/3] kylin git commit: stop testing storage type 0 as it is no longer used

2016-04-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 2f8df5160 -> 3aad1fc60


stop testing storage type 0 as it is no longer used


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/372d4335
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/372d4335
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/372d4335

Branch: refs/heads/yang-m1
Commit: 372d4335a0ef6cf913e68beaf599528f43eeb37c
Parents: 2f8df51
Author: Hongbin Ma 
Authored: Mon Apr 18 11:27:45 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 18 15:49:04 2016 +0800

--
 .../cube_desc/test_kylin_cube_with_slr_left_join_desc.json | 2 +-
 .../localmeta/cube_desc/test_kylin_cube_without_slr_desc.json  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/372d4335/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index bb59a05..9d82ac3 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -166,6 +166,6 @@
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : 0,
-  "storage_type" : 0,
+  "storage_type" : 2,
   "partition_date_start" : 0
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/372d4335/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index e1d740a..3741b22 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -330,6 +330,6 @@
   "auto_merge_time_ranges": null,
   "retention_range": 0,
   "engine_type": 2,
-  "storage_type": 0,
+  "storage_type": 2,
   "partition_date_start": 0
 }

kylin git commit: KYLIN-1591 enabling different block size setting for small column families

2016-04-18 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 41014d6a5 -> 22e6969cb


KYLIN-1591 enabling different block size setting for small column families


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22e6969c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22e6969c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22e6969c

Branch: refs/heads/master
Commit: 22e6969cb30f2146b23a9c81701b8a39dba15fd4
Parents: 41014d6
Author: Hongbin Ma 
Authored: Mon Apr 18 15:40:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 18 15:40:54 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  | 10 +++---
 .../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 +
 .../java/org/apache/kylin/gridtable/GTScanRequest.java |  3 ++-
 .../kylin/storage/hbase/steps/CubeHTableUtil.java  |  9 ++---
 4 files changed, 28 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 51aa8aa..4c7995b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -489,7 +489,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getQueryScanFuzzyKeyMax() {
-return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
 }
 
 public int getQueryStorageVisitScanRangeMax() {
@@ -558,8 +558,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.hbase.default.encoding", "FAST_DIFF");
 }
 
-public String getHbaseDefaultBlockSize() {
-return getOptional("kylin.hbase.default.block.size", "4194304");
+public int getHbaseDefaultBlockSize() {
+return Integer.valueOf(getOptional("kylin.hbase.default.block.size", 
"1048576"));
+}
+
+public int getHbaseSmallFamilyBlockSize() {
+return 
Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536"));
 }
 
 public boolean isHiveKeepFlatTable() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
index 5db2a9b..c5b2e19 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.model;
 
 import java.util.Arrays;
 
+import org.apache.kylin.metadata.model.MeasureDesc;
+
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc {
 return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + 
Arrays.toString(columns) + "]";
 }
 
+public boolean isMemoryHungry() {
+for (HBaseColumnDesc hBaseColumnDesc : columns) {
+for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) {
+if 
(measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
+return true;
+}
+}
+}
+return false;
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 5681057..bc478c9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -161,7 +161,8 @@ public class GTScanRequest {
 }
 
 if (!doAggr) {//Skip reading this section if you're not profiling! 
-int scanned = lookAndForget(result);
+int scanned = result.getScannedRowCount();
+

kylin git commit: stop testing storage type 0 as it is no longer used

2016-04-17 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 89fb4870c -> 41014d6a5


stop testing storage type 0 as it is no longer used


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41014d6a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41014d6a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41014d6a

Branch: refs/heads/master
Commit: 41014d6a5cc8d071738be97a97d28bf2c90e8621
Parents: 89fb487
Author: Hongbin Ma 
Authored: Mon Apr 18 13:56:35 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 18 13:56:40 2016 +0800

--
 .../java/org/apache/kylin/cube/common/RowKeySplitterTest.java  | 6 +++---
 .../test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java  | 2 +-
 .../test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java  | 3 +--
 .../cube_desc/test_kylin_cube_with_slr_left_join_desc.json | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/41014d6a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
index 1ddc745..b41f23b 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
@@ -48,7 +48,7 @@ public class RowKeySplitterTest extends 
LocalFileMetadataTestCase {
 
 RowKeySplitter rowKeySplitter = new 
RowKeySplitter(cube.getFirstSegment(), 11, 20);
 // base cuboid rowkey
-byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0,-104,-106,-128, 11, 
54, -105, 55,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 13, 71, 114, 65, 66, 73, 
78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
+byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 
11, 54, -105, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 
114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
 rowKeySplitter.split(input);
 
 assertEquals(11, rowKeySplitter.getBufferSize());
@@ -61,9 +61,9 @@ public class RowKeySplitterTest extends 
LocalFileMetadataTestCase {
 
 RowKeySplitter rowKeySplitter = new 
RowKeySplitter(cube.getFirstSegment(), 11, 20);
 // base cuboid rowkey
-byte[] input = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 
121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
+byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 
34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
 rowKeySplitter.split(input);
 
-assertEquals(9, rowKeySplitter.getBufferSize());
+assertEquals(10, rowKeySplitter.getBufferSize());
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/41014d6a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
index 0fe37f7..43498d3 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
@@ -53,7 +53,7 @@ public class RowKeyDecoderTest extends 
LocalFileMetadataTestCase {
 
 RowKeyDecoder rowKeyDecoder = new 
RowKeyDecoder(cube.getFirstSegment());
 
-byte[] key = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 
70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
+byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 
121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
 
 rowKeyDecoder.decode(key);
 List values = rowKeyDecoder.getValues();

http://git-wip-us.apache.org/repos/asf/kylin/blob/41014d6a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
index 1a6b4dd..ba71387 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
@@ -70,13 +70,12 @@ public class RowKeyEncoderTest extends 
LocalFileMetadataTestCase {
 
 byte[] encodedKey = rowKeyEncoder.encode(data);
 assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
-byte[] cuboidId =

kylin git commit: print coprocessor commit id even when exception

2016-04-15 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 6a638535e -> 2f8df5160


print coprocessor commit id even when exception


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f8df516
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f8df516
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f8df516

Branch: refs/heads/yang-m1
Commit: 2f8df516042638b71a79cea5ff4c98189240e9a8
Parents: 6a63853
Author: Hongbin Ma 
Authored: Fri Apr 15 15:51:23 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Apr 15 15:51:29 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f8df516/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 1e57633..f6476f8 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -288,7 +288,8 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 
 } catch (IOException ioe) {
 logger.error(ioe.toString());
-ResponseConverter.setControllerException(controller, ioe);
+IOException wrapped = new IOException("Error in coprocessor " + 
debugGitTag, ioe);
+ResponseConverter.setControllerException(controller, wrapped);
 } finally {
 IOUtils.closeQuietly(innerScanner);
 if (region != null) {

[2/2] kylin git commit: fix ci

2016-04-14 Thread mahongbin

fix ci


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d7a34d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d7a34d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d7a34d1

Branch: refs/heads/yang-m1
Commit: 9d7a34d16a9fd95d816e789a67c8927367078b6b
Parents: b114915
Author: Hongbin Ma 
Authored: Thu Apr 14 17:41:34 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 20:15:11 2016 +0800

--
 .../org/apache/kylin/gridtable/DictGridTableTest.java  | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9d7a34d1/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index b1b5ee9..56db01c 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -29,6 +29,7 @@ import java.util.List;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesSerializer;
 import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.gridtable.CubeCodeSystem;
 import org.apache.kylin.dict.NumberDictionaryBuilder;
@@ -52,13 +53,14 @@ import 
org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 
-public class DictGridTableTest {
+public class DictGridTableTest  extends LocalFileMetadataTestCase {
 
 private GridTable table;
 private GTInfo info;
@@ -74,8 +76,17 @@ public class DictGridTableTest {
 private CompareTupleFilter ageComp3;
 private CompareTupleFilter ageComp4;
 
+@After
+public void after() throws Exception {
+
+this.cleanupTestMetadata();
+}
+
 @Before
 public void setup() throws IOException {
+
+this.createTestMetadata();
+
 table = newTestTable();
 info = table.getInfo();

kylin git commit: fix ci

2016-04-14 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 9758c880d -> 9a913582b


fix ci


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a913582
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a913582
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a913582

Branch: refs/heads/master
Commit: 9a913582b05fde8a0c192bbeb1686f8a90e32f37
Parents: 9758c88
Author: Hongbin Ma 
Authored: Thu Apr 14 18:00:50 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 18:00:50 2016 +0800

--
 .../kylin/storage/hbase/steps/RangeKeyDistributionReducer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a913582/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
index c927ceb..b694f5f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionReducer.java
@@ -66,7 +66,7 @@ public class RangeKeyDistributionReducer extends 
KylinReducer

kylin git commit: fix ci

2016-04-14 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 71fccaedb -> 9758c880d


fix ci


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9758c880
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9758c880
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9758c880

Branch: refs/heads/master
Commit: 9758c880d4ddd951e402a02e67170de6c55e0712
Parents: 71fccae
Author: Hongbin Ma 
Authored: Thu Apr 14 17:41:34 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 17:41:34 2016 +0800

--
 .../org/apache/kylin/gridtable/DictGridTableTest.java  | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9758c880/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java 
b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index b411e96..e58371c 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -29,6 +29,7 @@ import java.util.List;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesSerializer;
 import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.gridtable.CubeCodeSystem;
 import org.apache.kylin.dict.NumberDictionaryBuilder;
@@ -52,13 +53,14 @@ import 
org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
 
-public class DictGridTableTest {
+public class DictGridTableTest  extends LocalFileMetadataTestCase {
 
 private GridTable table;
 private GTInfo info;
@@ -74,8 +76,17 @@ public class DictGridTableTest {
 private CompareTupleFilter ageComp3;
 private CompareTupleFilter ageComp4;
 
+@After
+public void after() throws Exception {
+
+this.cleanupTestMetadata();
+}
+
 @Before
 public void setup() throws IOException {
+
+this.createTestMetadata();
+
 table = newTestTable();
 info = table.getInfo();

kylin git commit: fix ci

2016-04-14 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 4a1bc1945 -> 71fccaedb


fix ci


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71fccaed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71fccaed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71fccaed

Branch: refs/heads/master
Commit: 71fccaedb2e9d5e288bef43e09050adb3b8757e7
Parents: 4a1bc19
Author: Hongbin Ma 
Authored: Thu Apr 14 16:37:46 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 16:39:44 2016 +0800

--
 .../job/impl/threadpool/DefaultScheduler.java   |   3 +-
 .../impl/threadpool/DefaultSchedulerTest.java   | 153 ++
 .../kylin/cube/DictionaryManagerTest.java   | 127 ---
 .../kylin/cube/ITDictionaryManagerTest.java | 127 +++
 .../impl/threadpool/ITDefaultSchedulerTest.java | 154 ---
 5 files changed, 282 insertions(+), 282 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/71fccaed/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 417e279..5e11041 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -182,9 +182,10 @@ public class DefaultScheduler implements 
Scheduler, Connecti
 logger.debug("Closing zk connection");
 try {
 shutdown();
-jobLock.unlock();
 } catch (SchedulerException e) {
 logger.error("error shutdown scheduler", e);
+} finally {
+jobLock.unlock();
 }
 }
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/71fccaed/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
new file mode 100644
index 000..df521f9
--- /dev/null
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -0,0 +1,153 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  * 
+ *  * http://www.apache.org/licenses/LICENSE-2.0
+ *  * 
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ * /
+ */
+
+package org.apache.kylin.job.impl.threadpool;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.job.BaseTestExecutable;
+import org.apache.kylin.job.ErrorTestExecutable;
+import org.apache.kylin.job.FailedTestExecutable;
+import org.apache.kylin.job.SelfStopExecutable;
+import org.apache.kylin.job.SucceedTestExecutable;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ */
+public class DefaultSchedulerTest extends BaseSchedulerTest {
+
+@Test
+public void testSingleTaskJob() throws Exception {
+DefaultChainedExecutable job = new DefaultChainedExecutable();
+BaseTestExecutable task1 = new SucceedTestExecutable();
+job.addTask(task1);
+jobService.addJob(job);
+waitForJobFinish(job.getId());
+Assert.assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(job.getId()).getState());
+

kylin git commit: KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable

2016-04-14 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 5042a30bc -> f3a33673f


KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f3a33673
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f3a33673
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f3a33673

Branch: refs/heads/master
Commit: f3a33673fcc39e3673a1b1e712fb44fcf781d8f5
Parents: 5042a30
Author: Hongbin Ma 
Authored: Thu Apr 14 15:06:31 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 15:06:31 2016 +0800

--
 .../kylin/gridtable/GTScanRangePlanner.java | 20 
 1 file changed, 12 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f3a33673/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 479064c..baf3428 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -63,9 +63,8 @@ public class GTScanRangePlanner {
 
 private static final Logger logger = 
LoggerFactory.getLogger(GTScanRangePlanner.class);
 
-private static final int MAX_HBASE_FUZZY_KEYS = 100;
-
 protected int maxScanRanges;
+protected int maxFuzzyKeys;
 
 //non-GT
 protected CubeSegment cubeSegment;
@@ -93,7 +92,8 @@ public class GTScanRangePlanner {
 public GTScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, 
TupleFilter filter, Set dimensions, Set groupbyDims, //
 Collection metrics) {
 
-this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
+this.maxFuzzyKeys = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
 
 this.cubeSegment = cubeSegment;
 this.cubeDesc = cubeSegment.getCubeDesc();
@@ -143,7 +143,9 @@ public class GTScanRangePlanner {
  */
 public GTScanRangePlanner(GTInfo info, Pair 
gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) {
 
-this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
+this.maxFuzzyKeys = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+
 this.gtInfo = info;
 
 IGTComparator comp = gtInfo.codeSystem.getComparator();
@@ -154,11 +156,13 @@ public class GTScanRangePlanner {
 //start key GTRecord compare to stop key GTRecord
 this.rangeStartEndComparator = getRangeStartEndComparator(comp);
 
+
 this.gtFilter = gtFilter;
 this.gtStartAndEnd = gtStartAndEnd;
 this.gtPartitionCol = gtPartitionCol;
 }
 
+
 public GTScanRequest planScanRequest(boolean allowPreAggregate) {
 GTScanRequest scanRequest;
 List scanRanges = this.planScanRanges();
@@ -308,8 +312,8 @@ public class GTScanRangePlanner {
 if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) 
{
 if 
(rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), 
range.end) <= 0 //
 && 
(rangeStartEndComparator.comparator.compare(range.begin, 
gtStartAndEnd.getSecond()) < 0 //
-|| 
rangeStartEndComparator.comparator.compare(range.begin, 
gtStartAndEnd.getSecond()) == 0 //
-&& (range.op == FilterOperatorEnum.EQ 
|| range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || 
range.op == FilterOperatorEnum.IN))) {
+|| 
rangeStartEndComparator.comparator.compare(range.begin, 
gtStartAndEnd.getSecond()) == 0 //
+&& (range.op == FilterOperatorEnum.EQ || range.op == 
FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == 
FilterOperatorEnum.IN))) {
 //segment range is [Closed,Open), but 
segmentStartAndEnd.getSecond() might be rounded, so use <= when has equals in 
condition. 
 } else {
 logger.debug("Pre-check partition col filter failed, 
partitionColRef {}, segment start {}, segment end {}, range begin {}, range end 
{}", //
@@ -346,7 +350,7 @@ public class GTScanRangePlanner {
 return result;
 }
 
-List>

kylin git commit: KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable

2016-04-13 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 5dfda003e -> 5042a30bc


KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5042a30b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5042a30b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5042a30b

Branch: refs/heads/master
Commit: 5042a30bcbc51a1061d2c09365c994dceb28d44e
Parents: 5dfda00
Author: Hongbin Ma 
Authored: Thu Apr 14 12:17:31 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 12:17:31 2016 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java   | 4 
 .../java/org/apache/kylin/gridtable/GTScanRangePlanner.java  | 8 +++-
 2 files changed, 7 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5042a30b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 712bcf7..51aa8aa 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -488,6 +488,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 setProperty("kylin.query.storage.visit.planner", v);
 }
 
+public int getQueryScanFuzzyKeyMax() {
+return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+}
+
 public int getQueryStorageVisitScanRangeMax() {
 return 
Integer.valueOf(this.getOptional("kylin.query.storage.visit.scanrange.max", 
"100"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5042a30b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 3f9bac0..479064c 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -32,6 +32,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.DateFormat;
@@ -62,7 +63,6 @@ public class GTScanRangePlanner {
 
 private static final Logger logger = 
LoggerFactory.getLogger(GTScanRangePlanner.class);
 
-private static final int MAX_SCAN_RANGES = 200;
 private static final int MAX_HBASE_FUZZY_KEYS = 100;
 
 protected int maxScanRanges;
@@ -93,7 +93,7 @@ public class GTScanRangePlanner {
 public GTScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, 
TupleFilter filter, Set dimensions, Set groupbyDims, //
 Collection metrics) {
 
-this.maxScanRanges = MAX_SCAN_RANGES;
+this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
 
 this.cubeSegment = cubeSegment;
 this.cubeDesc = cubeSegment.getCubeDesc();
@@ -143,7 +143,7 @@ public class GTScanRangePlanner {
  */
 public GTScanRangePlanner(GTInfo info, Pair 
gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) {
 
-this.maxScanRanges = MAX_SCAN_RANGES;
+this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
 this.gtInfo = info;
 
 IGTComparator comp = gtInfo.codeSystem.getComparator();
@@ -154,12 +154,10 @@ public class GTScanRangePlanner {
 //start key GTRecord compare to stop key GTRecord
 this.rangeStartEndComparator = getRangeStartEndComparator(comp);
 
-
 this.gtFilter = gtFilter;
 this.gtStartAndEnd = gtStartAndEnd;
 this.gtPartitionCol = gtPartitionCol;
 }
-
 
 public GTScanRequest planScanRequest(boolean allowPreAggregate) {
 GTScanRequest scanRequest;

kylin git commit: KYLIN-1584 Specify region cut size in cubedesc and leave the RealizationCapacity in model as a hint

2016-04-13 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 29240b7db -> 96eca30d8


KYLIN-1584 Specify region cut size in cubedesc and leave the 
RealizationCapacity in model as a hint


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96eca30d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96eca30d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96eca30d

Branch: refs/heads/yang-m1
Commit: 96eca30d8299c7a5d1f49102aca231c542c25020
Parents: 29240b7
Author: Hongbin Ma 
Authored: Thu Apr 14 12:03:19 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 12:03:31 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 33 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  8 +++
 .../storage/hbase/steps/CreateHTableJob.java|  8 +--
 .../hbase/steps/RangeKeyDistributionJob.java|  3 +-
 .../steps/RangeKeyDistributionReducer.java  |  4 +-
 .../kylin/storage/hbase/steps/RegionSize.java   | 51 
 6 files changed, 76 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/96eca30d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 314f24d..4a5106e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -400,25 +400,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", 
"300"));
 }
 
-public int getHBaseRegionCut(String capacity) {
-String cut;
-switch (capacity) {
-case "SMALL":
-cut = getOptional("kylin.hbase.region.cut.small", "10");
-break;
-case "MEDIUM":
-cut = getOptional("kylin.hbase.region.cut.medium", "20");
-break;
-case "LARGE":
-cut = getOptional("kylin.hbase.region.cut.large", "100");
-break;
-default:
-throw new IllegalArgumentException("Capacity not recognized: " + 
capacity);
-}
-
-return Integer.valueOf(cut);
-}
-
 public int getHBaseRegionCountMin() {
 return Integer.parseInt(getOptional("kylin.hbase.region.count.min", 
"1"));
 }
@@ -511,10 +492,20 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
 }
 
-public int getHBaseScanMaxResultSize() {
+public float getKylinHBaseRegionCutSmall() {
+return Float.valueOf(getOptional("kylin.hbase.region.cut.small", 
"10"));
+}
 
-return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
+public float getKylinHBaseRegionCutMedium() {
+return Float.valueOf(getOptional("kylin.hbase.region.cut.medium", 
"20"));
+}
+
+public float getKylinHBaseRegionCutLarge() {
+return Float.valueOf(getOptional("kylin.hbase.region.cut.large", 
"100"));
+}
 
+public int getHBaseScanMaxResultSize() {
+return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
 }
 
 public int getCubingInMemSamplingPercent() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/96eca30d/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 240cf52..9323990 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -145,6 +145,9 @@ public class CubeDesc extends RootPersistentEntity {
 private long[] autoMergeTimeRanges;
 @JsonProperty("retention_range")
 private long retentionRange = 0;
+
+@JsonProperty("region_size")
+private float regionSize = 
0;//https://issues.apache.org/jira/browse/KYLIN-1584
 
 @JsonProperty("engine_type")
 private int engineType = IEngineAware.ID_MR_V1;
@@ -410,6 +413,11 @@ public class CubeDesc extends RootPersistentEntity {
 this.overrideKylinProps = overrideKylinProps;
 }
 
+public float getRegionSize() {
+return regionSize;
+}
+
+
 @Override
 public boolean equals(Object o) {
 if (this == o)

kylin git commit: test

2016-04-13 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/mhb_test [created] 6ca8f30e9


test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ca8f30e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ca8f30e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ca8f30e

Branch: refs/heads/mhb_test
Commit: 6ca8f30e95ef470b5721a16e315c14d5253e356c
Parents: 1d4c7bb
Author: Hongbin Ma 
Authored: Wed Apr 13 23:23:43 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 13 23:23:43 2016 +0800

--
 .../apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java   | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6ca8f30e/kylin-it/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
index ad1ddd3..3f07843 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
@@ -38,10 +38,12 @@ import org.apache.kylin.job.SucceedTestExecutable;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 import org.apache.kylin.job.execution.ExecutableState;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /**
  */
+@Ignore
 public class ITDefaultSchedulerTest extends BaseSchedulerTest {
 
 @Test

[8/8] kylin git commit: update kylin metadata version from 1.5.1 to 1.5.2

2016-04-12 Thread mahongbin

update kylin metadata version from 1.5.1 to 1.5.2


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59cb57ca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59cb57ca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59cb57ca

Branch: refs/heads/master
Commit: 59cb57ca6e1a545628d0057d5980c4fe7e789536
Parents: b6c893d
Author: Hongbin Ma 
Authored: Wed Apr 13 11:39:42 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 13 11:39:42 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinVersion.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59cb57ca/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index d55f969..0700968 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -59,7 +59,7 @@ public class KylinVersion {
 /**
  * Require MANUAL updating kylin version per ANY upgrading.
  */
-private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("1.5.1");
+private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("1.5.2");
 
 private static final Set SIGNATURE_INCOMPATIBLE_REVISIONS = 
new HashSet();

[1/8] kylin git commit: minor bug fix in removing ext filter from project

2016-04-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master aaf3b870c -> 59cb57ca6


minor bug fix in removing ext filter from project


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6c893db
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6c893db
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6c893db

Branch: refs/heads/master
Commit: b6c893dbeefead2523fc686055736526d4f3ca3e
Parents: b2a3861
Author: Hongbin Ma 
Authored: Tue Apr 12 16:01:45 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 13 11:11:15 2016 +0800

--
 .../java/org/apache/kylin/metadata/MetadataManager.java | 12 
 .../apache/kylin/metadata/project/ProjectInstance.java  |  9 +
 2 files changed, 9 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6c893db/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index aa249fd..3391ef4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -143,7 +143,7 @@ public class MetadataManager {
 return Lists.newArrayList(srcTableMap.values());
 }
 
-public List listAllExternalFilters(){
+public List listAllExternalFilters() {
 return Lists.newArrayList(extFilterMap.values());
 }
 
@@ -217,27 +217,23 @@ public class MetadataManager {
 }
 
 public void saveExternalFilter(ExternalFilterDesc desc) throws IOException 
{
-if(desc.getUuid() == null){
+if (desc.getUuid() == null) {
 throw new IllegalArgumentException("UUID not set.");
 }
 String path = desc.getResourcePath();
-getStore().putResource(path,desc,EXTERNAL_FILTER_DESC_SERIALIZER);
+getStore().putResource(path, desc, EXTERNAL_FILTER_DESC_SERIALIZER);
 desc = reloadExternalFilterAt(path);
-extFilterMap.put(desc.getName(),desc);
+extFilterMap.put(desc.getName(), desc);
 
 }
 
 public void removeExternalFilter(String name) throws IOException {
-if(name !=null ){
-name = name.toLowerCase();
-}
 String path = ExternalFilterDesc.concatResourcePath(name);
 getStore().deleteResource(path);
 extFilterMap.remove(name);
 
 }
 
-
 private void init(KylinConfig config) throws IOException {
 this.config = config;
 this.srcTableMap = new CaseInsensitiveStringCache<>(config, 
Broadcaster.TYPE.TABLE);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6c893db/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
index 9567da3..74f843f 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
@@ -210,9 +210,13 @@ public class ProjectInstance extends RootPersistentEntity {
 public void removeTable(String tableName) {
 tables.remove(tableName.toUpperCase());
 }
+
+public void addExtFilter(String extFilterName){
+this.getExtFilters().add(extFilterName);
+}
 
 public void removeExtFilter(String filterName) {
-extFilters.remove(filterName.toUpperCase());
+extFilters.remove(filterName);
 }
 
 public int getTablesCount() {
@@ -227,9 +231,6 @@ public class ProjectInstance extends RootPersistentEntity {
 return tables;
 }
 
-public void addExtFilter(String extFilterName){
-this.getExtFilters().add(extFilterName.toLowerCase());
-}
 
 public Set getExtFilters(){
 return extFilters;

[5/8] kylin git commit: refactor

2016-04-12 Thread mahongbin

refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b26b2489
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b26b2489
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b26b2489

Branch: refs/heads/master
Commit: b26b2489baddcfd148eaf8f17330878bbf349048
Parents: aaf3b87
Author: Hongbin Ma 
Authored: Mon Apr 11 17:33:07 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 13 11:11:15 2016 +0800

--
 .../kylin/job/DeployLocalMetaToRemoteTest.java  |   4 +-
 .../java/org/apache/kylin/job/DeployUtil.java   |   4 +-
 .../org/apache/kylin/common/util/ByteArray.java |  11 +-
 .../kylin/common/util/Log4jConfigurer.java  |   2 +
 .../common/util/AbstractKylinTestCase.java  |  14 +-
 .../common/util/HBaseMetadataTestCase.java  |   4 +-
 .../common/util/LocalFileMetadataTestCase.java  |   8 +-
 .../org/apache/kylin/cube/cuboid/Cuboid.java|  18 ++
 .../apache/kylin/cube/kv/LazyRowKeyEncoder.java |  25 --
 .../org/apache/kylin/cube/model/CubeDesc.java   |   5 +
 .../org/apache/kylin/gridtable/GTRecord.java|  10 +-
 .../org/apache/kylin/gridtable/GTScanRange.java |   2 -
 .../kylin/gridtable/GTScanRangePlanner.java | 294 ---
 .../apache/kylin/gridtable/GTScanRequest.java   |  68 +++--
 .../org/apache/kylin/gridtable/IGTStorage.java  |  27 ++
 .../apache/kylin/gridtable/ScannerWorker.java   |  72 +
 .../kylin/cube/AggregationGroupRuleTest.java|  19 +-
 .../apache/kylin/cube/RowKeyAttrRuleTest.java   |   5 +-
 .../cube/inmemcubing/DoggedCubeBuilderTest.java | 160 --
 .../cube/inmemcubing/InMemCubeBuilderTest.java  | 268 -
 .../kylin/gridtable/DictGridTableTest.java  |  58 ++--
 .../impl/threadpool/DefaultSchedulerTest.java   | 151 --
 .../filter/UDF/MassInValueProviderFactory.java  |   1 -
 .../apache/kylin/storage/StorageMockUtils.java  | 189 
 .../kylin/storage/cache/StorageMockUtils.java   | 157 --
 kylin-it/pom.xml|   7 +
 .../ITDoggedCubeBuilderStressTest.java  |   8 +-
 .../inmemcubing/ITDoggedCubeBuilderTest.java| 163 ++
 .../inmemcubing/ITInMemCubeBuilderTest.java | 271 +
 .../impl/threadpool/ITDefaultSchedulerTest.java | 154 ++
 .../kylin/provision/BuildCubeWithEngine.java|   4 +-
 .../kylin/provision/BuildCubeWithSpark.java |   6 +-
 .../kylin/provision/BuildCubeWithStream.java|   4 +-
 .../kylin/provision/BuildIIWithEngine.java  |   4 +-
 .../kylin/provision/BuildIIWithStream.java  |   2 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  11 -
 .../kylin/storage/hbase/ITStorageTest.java  |   2 +-
 source-hive/pom.xml |   7 +
 .../kylin/source/hive/HiveCmdBuilderTest.java   |   6 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 111 ---
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |  56 +---
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  94 --
 .../hbase/cube/v2/CubeSegmentScanner.java   | 215 +-
 .../storage/hbase/cube/v2/CubeStorageQuery.java |  30 +-
 .../kylin/storage/hbase/cube/v2/RawScan.java|  10 +
 .../coprocessor/endpoint/CubeVisitService.java  |  88 --
 .../hbase/steps/SandboxMetastoreCLI.java|   2 +-
 47 files changed, 1539 insertions(+), 1292 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b26b2489/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
index 1267ab7..8494607 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
@@ -45,7 +45,7 @@ public class DeployLocalMetaToRemoteTest {
 public static void beforeClass() throws Exception {
 logger.info("Adding to classpath: " + new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
 ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
-System.setProperty(KylinConfig.KYLIN_CONF, 
"../examples/test_case_data/sandbox");
+System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
 throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
 }
@@ -53,7 +53,7 @@ public class DeployLocalMetaToRemoteTest {
 
 @Before

[3/8] kylin git commit: refactor

2016-04-12 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/b26b2489/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index ef53cb7..938145b 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -19,30 +19,35 @@
 package org.apache.kylin.storage.hbase.cube.v2;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
-import javax.annotation.Nullable;
-
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.ShardingHash;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.gridtable.GTInfo;
 import org.apache.kylin.gridtable.GTRecord;
 import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.IGTScanner;
 import org.apache.kylin.gridtable.IGTStore;
+import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.storage.hbase.HBaseConnection;
+import 
org.apache.kylin.storage.hbase.cube.v2.filter.MassInValueProviderFactoryImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Function;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 
@@ -84,53 +89,77 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
 }
 }
 
-public CubeHBaseScanRPC(CubeSegment cubeSeg, Cuboid cuboid, GTInfo 
fullGTInfo) {
+public CubeHBaseScanRPC(CubeSegment cubeSeg, Cuboid cuboid, final GTInfo 
fullGTInfo) {
 super(cubeSeg, cuboid, fullGTInfo);
+MassInTupleFilter.VALUE_PROVIDER_FACTORY = new 
MassInValueProviderFactoryImpl(new MassInValueProviderFactoryImpl.DimEncAware() 
{
+@Override
+public DimensionEncoding getDimEnc(TblColRef col) {
+return 
fullGTInfo.getCodeSystem().getDimEnc(col.getColumnDesc().getZeroBasedIndex());
+}
+});
 }
 
 @Override
-public IGTScanner getGTScanner(final List scanRequests) 
throws IOException {
-final List scanners = Lists.newArrayList();
-for (GTScanRequest request : scanRequests) {
-scanners.add(getGTScanner(request));
-}
+public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws 
IOException {
+final IGTScanner scanner = getGTScannerInternal(scanRequest);
 
 return new IGTScanner() {
 @Override
 public GTInfo getInfo() {
-return scanners.get(0).getInfo();
+return scanner.getInfo();
 }
 
 @Override
 public int getScannedRowCount() {
 int sum = 0;
-for (IGTScanner s : scanners) {
-sum += s.getScannedRowCount();
-}
+sum += scanner.getScannedRowCount();
 return sum;
 }
 
 @Override
 public void close() throws IOException {
-for (IGTScanner s : scanners) {
-s.close();
-}
+scanner.close();
 }
 
 @Override
 public Iterator iterator() {
-return 
Iterators.concat(Iterators.transform(scanners.iterator(), new 
Function() {
-@Nullable
-@Override
-public Iterator apply(IGTScanner input) {
-return input.iterator();
-}
-}));
+return scanner.iterator();
 }
 };
 }
 
-private IGTScanner getGTScanner(final GTScanRequest scanRequest) throws 
IOException {
+//for non-sharding cases it will only return one byte[] with not shard at 
beginning
+private List getRowKeysDifferentShards(byte[] halfCookedKey) {
+final short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());
+
+if (!cubeSeg.isEnableSharding()) {
+return Lists.newArrayList(halfCookedKey);//not shard to append at 
head,

[6/8] kylin git commit: KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout

2016-04-12 Thread mahongbin

KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11be1e38
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11be1e38
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11be1e38

Branch: refs/heads/master
Commit: 11be1e3826cdea8db8df8975ebdff5cf1d93444f
Parents: b26b248
Author: Hongbin Ma 
Authored: Tue Apr 12 09:47:52 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 13 11:11:15 2016 +0800

--
 .../apache/kylin/gridtable/GTScanRequest.java   |   2 +
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  16 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  58 ++-
 .../endpoint/generated/CubeVisitProtos.java | 436 +--
 .../endpoint/protobuf/CubeVisit.proto   |   3 +
 5 files changed, 463 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/11be1e38/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index c4abb57..5681057 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.BytesSerializer;
 import org.apache.kylin.common.util.BytesUtil;
@@ -187,6 +188,7 @@ public class GTScanRequest {
 }
 }
 System.out.println("Meaningless byte is " + meaninglessByte);
+IOUtils.closeQuietly(scanner);
 return scanned;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/11be1e38/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 1d3da28..38041b3 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -162,6 +162,10 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 throw new RuntimeException("error when waiting queue", e);
 }
 }
+
+public long getTimeout() {
+return timeout;
+}
 }
 
 static class EndpointResultsAsGTScanner implements IGTScanner {
@@ -313,7 +317,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 logger.debug("Serialized scanRequestBytes {} bytes, rawScanBytesString 
{} bytes", scanRequestByteString.size(), rawScanByteString.size());
 
-
 logger.info("The scan {} for segment {} is as below, shard part of 
start/end key is set to 0", 
Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg);
 for (RawScan rs : rawScans) {
 logScan(rs, cubeSeg.getStorageLocationIdentifier());
@@ -323,7 +326,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 final AtomicInteger totalScannedCount = new AtomicInteger(0);
 final ExpectedSizeIterator epResultItr = new 
ExpectedSizeIterator(shardNum);
-final String currentThreadName = Thread.currentThread().getName();
 
 for (final Pair epRange : 
getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
 final ByteString finalScanRequestByteString = 
scanRequestByteString;
@@ -338,6 +340,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 }
 
builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize());
 builder.setBehavior(toggle);
+builder.setStartTime(System.currentTimeMillis());
+builder.setTimeout(epResultItr.getTimeout());
 
 Map results;
 try {
@@ -348,7 +352,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 for (Map.Entry 
result : results.entrySet()) {
 
totalScannedCount.addAndGet(result.getValue().getStats().getScannedRowCount());
-logger.info(" " +

[4/8] kylin git commit: refactor

2016-04-12 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/b26b2489/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
deleted file mode 100644
index d50baad..000
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.impl.threadpool;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kylin.job.BaseTestExecutable;
-import org.apache.kylin.job.ErrorTestExecutable;
-import org.apache.kylin.job.FailedTestExecutable;
-import org.apache.kylin.job.SelfStopExecutable;
-import org.apache.kylin.job.SucceedTestExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.junit.Test;
-
-/**
- */
-public class DefaultSchedulerTest extends BaseSchedulerTest {
-
-@Test
-public void testSingleTaskJob() throws Exception {
-DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SucceedTestExecutable();
-job.addTask(task1);
-jobService.addJob(job);
-waitForJobFinish(job.getId());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(job.getId()).getState());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(task1.getId()).getState());
-}
-
-@Test
-public void testSucceed() throws Exception {
-DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SucceedTestExecutable();
-BaseTestExecutable task2 = new SucceedTestExecutable();
-job.addTask(task1);
-job.addTask(task2);
-jobService.addJob(job);
-waitForJobFinish(job.getId());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(job.getId()).getState());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(task1.getId()).getState());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(task2.getId()).getState());
-}
-
-@Test
-public void testSucceedAndFailed() throws Exception {
-DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SucceedTestExecutable();
-BaseTestExecutable task2 = new FailedTestExecutable();
-job.addTask(task1);
-job.addTask(task2);
-jobService.addJob(job);
-waitForJobFinish(job.getId());
-assertEquals(ExecutableState.ERROR, 
jobService.getOutput(job.getId()).getState());
-assertEquals(ExecutableState.SUCCEED, 
jobService.getOutput(task1.getId()).getState());
-assertEquals(ExecutableState.ERROR, 
jobService.getOutput(task2.getId()).getState());
-}
-
-@Test
-public void testSucceedAndError() throws Exception {
-DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new ErrorTestExecutable();
-BaseTestExecutable task2 = new SucceedTestExecutable();
-job.addTask(task1);
-job.addTask(task2);
-jobService.addJob(job);
-waitForJobFinish(job.getId());
-assertEquals(ExecutableState.ERROR, 
jobService.getOutput(job.getId()).getState());
-assertEquals(ExecutableState.ERROR, 
jobService.getOutput(task1.getId()).getState());
-assertEquals(ExecutableState.READY, 
jobService.getOutput(task2.getId()).getState());
-}
-
-@Test
-public void testDiscard() throws Exception {
-DefaultChainedExecutable job = new

kylin git commit: KYLIN-1556 remove the left hybrid and refine candidate priority for IT

2016-04-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master ec2845af4 -> aaf3b870c


KYLIN-1556 remove the left hybrid and refine candidate priority for IT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aaf3b870
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aaf3b870
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aaf3b870

Branch: refs/heads/master
Commit: aaf3b870c41cf356152e25335ecb089ee1a72223
Parents: ec2845a
Author: Hongbin Ma 
Authored: Wed Apr 13 11:09:55 2016 +0800
Committer: Hongbin Ma 
Committed: Wed Apr 13 11:10:14 2016 +0800

--
 .../apache/kylin/cube/CubeCapabilityChecker.java |  2 +-
 .../kylin/metadata/model/FunctionDesc.java   | 12 +++-
 .../metadata/realization/CapabilityResult.java   |  2 +-
 .../hybrid/test_kylin_hybrid_left_join.json  | 14 --
 .../localmeta/project/default.json   |  5 -
 .../apache/kylin/query/ITCombinationTest.java| 12 
 .../org/apache/kylin/query/ITIIQueryTest.java|  8 +---
 .../org/apache/kylin/query/ITKylinQueryTest.java | 11 +++
 .../apache/kylin/query/routing/Candidate.java| 19 ---
 9 files changed, 49 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aaf3b870/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e21dc2b..bd45d07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -177,7 +177,7 @@ public class CubeCapabilityChecker {
 
 // calcite can do aggregation from columns on-the-fly
 List neededCols = 
functionDesc.getParameter().getColRefs();
-if (neededCols.size() > 0 && 
cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols)) {
+if (neededCols.size() > 0 && 
cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols) && 
FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
 result.influences.add(new 
CapabilityResult.DimensionAsMeasure(functionDesc));
 it.remove();
 continue;

http://git-wip-us.apache.org/repos/asf/kylin/blob/aaf3b870/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 9e3effb..f3a81d6 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.MeasureTypeFactory;
@@ -31,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 /**
  */
@@ -41,6 +43,14 @@ public class FunctionDesc {
 public static final String FUNC_MIN = "MIN";
 public static final String FUNC_MAX = "MAX";
 public static final String FUNC_COUNT = "COUNT";
+public static final Set BUILT_IN_AGGREGATIONS = Sets.newHashSet();
+
+static {
+BUILT_IN_AGGREGATIONS.add(FUNC_COUNT);
+BUILT_IN_AGGREGATIONS.add(FUNC_MAX);
+BUILT_IN_AGGREGATIONS.add(FUNC_MIN);
+BUILT_IN_AGGREGATIONS.add(FUNC_SUM);
+}
 
 public static final String PARAMETER_TYPE_CONSTANT = "constant";
 public static final String PARAMETER_TYPE_COLUMN = "column";
@@ -67,7 +77,7 @@ public class FunctionDesc {
 ArrayList colRefs = Lists.newArrayList();
 for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) 
{
 if (p.isColumnType()) {
-ColumnDesc sourceColumn = 
findColumn(factTable,lookupTables,p.getValue());
+ColumnDesc sourceColumn = findColumn(factTable, lookupTables, 
p.getValue());
 TblColRef colRef = new TblColRef(sourceColumn);
 colRefs.add(colRef);
 }

kylin git commit: minor bug fix in removing ext filter from project

2016-04-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 c1af3c015 -> 71113643f


minor bug fix in removing ext filter from project


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71113643
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71113643
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71113643

Branch: refs/heads/yang-m1
Commit: 71113643fc3d3137e0baf3edbba513ee0fd47820
Parents: c1af3c0
Author: Hongbin Ma 
Authored: Tue Apr 12 16:04:32 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 16:04:32 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/MetadataManager.java | 3 ---
 .../java/org/apache/kylin/metadata/project/ProjectInstance.java  | 4 ++--
 2 files changed, 2 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/71113643/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 9ed1f13..26ee9c4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -228,9 +228,6 @@ public class MetadataManager {
 }
 
 public void removeExternalFilter(String name) throws IOException {
-if(name !=null ){
-name = name.toLowerCase();
-}
 String path = ExternalFilterDesc.concatResourcePath(name);
 getStore().deleteResource(path);
 extFilterMap.remove(name);

http://git-wip-us.apache.org/repos/asf/kylin/blob/71113643/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
index 9567da3..b488d07 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
@@ -212,7 +212,7 @@ public class ProjectInstance extends RootPersistentEntity {
 }
 
 public void removeExtFilter(String filterName) {
-extFilters.remove(filterName.toUpperCase());
+extFilters.remove(filterName);
 }
 
 public int getTablesCount() {
@@ -228,7 +228,7 @@ public class ProjectInstance extends RootPersistentEntity {
 }
 
 public void addExtFilter(String extFilterName){
-this.getExtFilters().add(extFilterName.toLowerCase());
+this.getExtFilters().add(extFilterName);
 }
 
 public Set getExtFilters(){

kylin git commit: KYLIN-1579 bug fix

2016-04-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 d291e8473 -> c1af3c015


KYLIN-1579 bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1af3c01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1af3c01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1af3c01

Branch: refs/heads/yang-m1
Commit: c1af3c015549faacd7968a54a8ef14c767f6b83b
Parents: d291e84
Author: Hongbin Ma 
Authored: Tue Apr 12 15:49:09 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 15:49:09 2016 +0800

--
 .../storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af3c01/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 554a764..cf39641 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -333,12 +333,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 if 
(result.getValue().getStats().getNormalComplete() != 1) {
 abnormalFinish = true;
-}
-
-try {
-
epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows(;
-} catch (IOException | DataFormatException e) {
-throw new RuntimeException(logHeader + "Error 
when decompressing", e);
+} else {
+try {
+
epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows(;
+} catch (IOException | DataFormatException e) {
+throw new RuntimeException(logHeader + 
"Error when decompressing", e);
+}
 }
 }

kylin git commit: KYLIN-1579 revise logging

2016-04-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 c17bb7366 -> d291e8473


KYLIN-1579 revise logging


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d291e847
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d291e847
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d291e847

Branch: refs/heads/yang-m1
Commit: d291e847388ab1c12c1a6c1b330eb8d5ba7fb0f4
Parents: c17bb73
Author: Hongbin Ma 
Authored: Tue Apr 12 15:31:31 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 15:31:31 2016 +0800

--
 .../storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 16 ++--
 1 file changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d291e847/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index bfc240a..554a764 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -161,7 +161,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 }
 }
 
-
 public long getTimeout() {
 return timeout;
 }
@@ -318,29 +317,34 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 builder.setBehavior(toggle);
 builder.setStartTime(System.currentTimeMillis());
 builder.setTimeout(epResultItr.getTimeout());
-
+String logHeader = " ";
 
 Map results;
 try {
 results = getResults(builder.build(), 
conn.getTable(cubeSeg.getStorageLocationIdentifier()), epRange.getFirst(), 
epRange.getSecond());
 } catch (Throwable throwable) {
-throw new RuntimeException(" " + 
"Error when visiting cubes by endpoint", throwable);
+throw new RuntimeException(logHeader + "Error when 
visiting cubes by endpoint", throwable);
 }
 
+boolean abnormalFinish = false;
 for (Map.Entry result : results.entrySet()) {
 
totalScannedCount.addAndGet(result.getValue().getStats().getScannedRowCount());
-logger.info(" " + 
getStatsString(result));
+logger.info(logHeader + getStatsString(result));
 
 if 
(result.getValue().getStats().getNormalComplete() != 1) {
-throw new RuntimeException("The coprocessor 
thread stopped itself due to scan timeout.");
+abnormalFinish = true;
 }
 
 try {
 
epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows(;
 } catch (IOException | DataFormatException e) {
-throw new RuntimeException("Error when 
decompressing", e);
+throw new RuntimeException(logHeader + "Error 
when decompressing", e);
 }
 }
+
+if (abnormalFinish) {
+throw new RuntimeException(logHeader + "The 
coprocessor thread stopped itself due to scan timeout.");
+}
 }
 }
 });

kylin git commit: KYLIN-1597 consider clock gap between query server and region server

2016-04-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 3a8a83b2a -> c17bb7366


KYLIN-1597 consider clock gap between query server and region server


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c17bb736
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c17bb736
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c17bb736

Branch: refs/heads/yang-m1
Commit: c17bb7366f242834d1cb89091205da80357c8021
Parents: 3a8a83b
Author: Hongbin Ma 
Authored: Tue Apr 12 14:17:56 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 14:17:56 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c17bb736/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 9dfc607..753e9d3 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -216,7 +216,7 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 }
 
 final MutableBoolean normalComplete = new MutableBoolean(true);
-final long startTime = request.getStartTime();
+final long startTime = 
this.serviceStartTime;//request.getStartTime();
 final long timeout = (long) (request.getTimeout() * 0.95);
 InnerScannerAsIterator cellListIterator = new 
InnerScannerAsIterator(innerScanner, normalComplete, startTime, timeout);

kylin git commit: KYLIN-1579 IT preparation classes like BuildCubeWithEngine should exit with status code upon build exception

2016-04-11 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 bb1b6c3f9 -> 3a8a83b2a


KYLIN-1579 IT preparation classes like BuildCubeWithEngine should exit with 
status code upon build exception


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a8a83b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a8a83b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a8a83b2

Branch: refs/heads/yang-m1
Commit: 3a8a83b2a90a7b234ba75e9f227c83164294eca6
Parents: bb1b6c3
Author: Hongbin Ma 
Authored: Tue Apr 12 11:35:54 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 11:40:33 2016 +0800

--
 .../kylin/provision/BuildCubeWithEngine.java| 23 +++
 .../kylin/provision/BuildCubeWithStream.java| 24 
 .../kylin/provision/BuildIIWithStream.java  | 21 ++---
 3 files changed, 42 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a8a83b2/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index cfefef3..942692a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -76,15 +76,20 @@ public class BuildCubeWithEngine {
 private static final Log logger = 
LogFactory.getLog(BuildCubeWithEngine.class);
 
 public static void main(String[] args) throws Exception {
-beforeClass();
-
-BuildCubeWithEngine buildCubeWithEngine = new BuildCubeWithEngine();
-buildCubeWithEngine.before();
-buildCubeWithEngine.build();
-logger.info("Build is done");
-afterClass();
-logger.info("Going to exit");
-System.exit(0);
+try {
+beforeClass();
+
+BuildCubeWithEngine buildCubeWithEngine = new 
BuildCubeWithEngine();
+buildCubeWithEngine.before();
+buildCubeWithEngine.build();
+logger.info("Build is done");
+afterClass();
+logger.info("Going to exit");
+System.exit(0);
+} catch (Exception e) {
+logger.error("error", e);
+System.exit(1);
+}
 }
 
 public static void beforeClass() throws Exception {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a8a83b2/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index eeff999..b48fde7 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -53,15 +53,21 @@ public class BuildCubeWithStream {
 private KylinConfig kylinConfig;
 
 public static void main(String[] args) throws Exception {
-beforeClass();
-BuildCubeWithStream buildCubeWithStream = new BuildCubeWithStream();
-buildCubeWithStream.before();
-buildCubeWithStream.build();
-logger.info("Build is done");
-afterClass();
-logger.info("Going to exit");
-System.exit(0);
-
+try {
+beforeClass();
+
+BuildCubeWithStream buildCubeWithStream = new 
BuildCubeWithStream();
+buildCubeWithStream.before();
+buildCubeWithStream.build();
+logger.info("Build is done");
+afterClass();
+logger.info("Going to exit");
+System.exit(0);
+} catch (Exception e) {
+logger.error("error", e);
+System.exit(1);
+}
+
 }
 
 public static void beforeClass() throws Exception {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3a8a83b2/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
index 9b7cd14..a77a1c4 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
@@ -93,14 +93,19 @@ public class BuildIIWithStream {
 private KylinConfig kylinConfig;
 
 public static void

kylin git commit: KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout

2016-04-11 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 3eeabeb51 -> bb1b6c3f9


 KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bb1b6c3f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bb1b6c3f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bb1b6c3f

Branch: refs/heads/yang-m1
Commit: bb1b6c3f97e36e1b473c0ba33c6544c1debcacee
Parents: 3eeabeb
Author: Hongbin Ma 
Authored: Tue Apr 12 10:32:32 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 10:32:32 2016 +0800

--
 .../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1b6c3f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 0879661..9dfc607 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -250,10 +250,10 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 byte[] compressedAllRows;
 if (normalComplete.booleanValue()) {
 allRows = outputStream.toByteArray();
-compressedAllRows = CompressionUtils.compress(allRows);
 } else {
-compressedAllRows = new byte[0];
+allRows = new byte[0];
 }
+compressedAllRows = CompressionUtils.compress(allRows);
 
 
 appendProfileInfo(sb, "compress done");

kylin git commit: KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout

2016-04-11 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 9513b8e47 -> 3eeabeb51


 KYLIN-1578 Coprocessor thread voluntarily stop itself when it reaches timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3eeabeb5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3eeabeb5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3eeabeb5

Branch: refs/heads/yang-m1
Commit: 3eeabeb519d39fbfd5805b41faaf3d94013f1c29
Parents: 9513b8e
Author: Hongbin Ma 
Authored: Tue Apr 12 10:30:39 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Apr 12 10:30:39 2016 +0800

--
 .../cube/v2/coprocessor/endpoint/CubeVisitService.java| 10 --
 1 file changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3eeabeb5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 596814f..0879661 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -247,8 +247,14 @@ public class CubeVisitService extends 
CubeVisitProtos.CubeVisitService implement
 appendProfileInfo(sb, "agg done");
 
 //outputStream.close() is not necessary
-allRows = outputStream.toByteArray();
-byte[] compressedAllRows = CompressionUtils.compress(allRows);
+byte[] compressedAllRows;
+if (normalComplete.booleanValue()) {
+allRows = outputStream.toByteArray();
+compressedAllRows = CompressionUtils.compress(allRows);
+} else {
+compressedAllRows = new byte[0];
+}
+
 
 appendProfileInfo(sb, "compress done");

kylin git commit: revise hdfs filter cache

2016-04-11 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 4d16e4d0e -> 737ba33b6


revise hdfs filter cache


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/737ba33b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/737ba33b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/737ba33b

Branch: refs/heads/master
Commit: 737ba33b6be4e69ab6533c03e1bb3ba5aeb776a4
Parents: 4d16e4d
Author: Hongbin Ma 
Authored: Mon Apr 11 18:07:20 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 11 18:08:13 2016 +0800

--
 .../cube/v2/filter/MassInValueProviderImpl.java | 57 ++--
 1 file changed, 30 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/737ba33b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
index 16157ec..525645d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
@@ -46,7 +46,7 @@ import com.google.common.collect.Sets;
 public class MassInValueProviderImpl implements MassInValueProvider {
 public static final Logger logger = 
LoggerFactory.getLogger(MassInValueProviderImpl.class);
 
-private final static Cache> hdfs_caches 
= CacheBuilder.newBuilder().maximumSize(10).weakValues().removalListener(new 
RemovalListener() {
+private final static Cache> hdfs_caches 
= CacheBuilder.newBuilder().maximumSize(3).removalListener(new 
RemovalListener() {
 @Override
 public void onRemoval(RemovalNotification 
notification) {
 logger.debug(String.valueOf(notification.getCause()));
@@ -64,39 +64,42 @@ public class MassInValueProviderImpl implements 
MassInValueProvider {
 
 FileSystem fileSystem = null;
 try {
-fileSystem = FileSystem.get(HBaseConfiguration.create());
-
-long modificationTime = fileSystem.getFileStatus(new 
Path(filterResourceIdentifier)).getModificationTime();
-Pair cached = 
hdfs_caches.getIfPresent(filterResourceIdentifier);
-if (cached != null && 
cached.getFirst().equals(modificationTime)) {
-ret = cached.getSecond();
-logger.info("Load HDFS from cache using " + 
stopwatch.elapsedMillis() + " millis");
-return;
-}
+synchronized (hdfs_caches) {
+
+fileSystem = FileSystem.get(HBaseConfiguration.create());
+
+long modificationTime = fileSystem.getFileStatus(new 
Path(filterResourceIdentifier)).getModificationTime();
+Pair cached = 
hdfs_caches.getIfPresent(filterResourceIdentifier);
+if (cached != null && 
cached.getFirst().equals(modificationTime)) {
+ret = cached.getSecond();
+logger.info("Load HDFS from cache using " + 
stopwatch.elapsedMillis() + " millis");
+return;
+}
 
-InputStream inputStream = fileSystem.open(new 
Path(filterResourceIdentifier));
-List lines = IOUtils.readLines(inputStream);
+InputStream inputStream = fileSystem.open(new 
Path(filterResourceIdentifier));
+List lines = IOUtils.readLines(inputStream);
 
-logger.info("Load HDFS finished after " + 
stopwatch.elapsedMillis() + " millis");
+logger.info("Load HDFS finished after " + 
stopwatch.elapsedMillis() + " millis");
 
-for (String line : lines) {
-if (StringUtils.isEmpty(line)) {
-continue;
-}
+for (String line : lines) {
+if (StringUtils.isEmpty(line)) {
+continue;
+}
 
-try {
-ByteArray byteArray = 
ByteArray.allocate(encoding.getLengthOfEncoding());
-encoding.encode(line.getBytes(), 
line.getBytes().length, byteArray.array(), 0);
-ret.add(byteArray);
-} catch (Exception e) {
-

kylin git commit: add cache to hdfs filter

2016-04-11 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 36c76427a -> 7036e4b8f


add cache to hdfs filter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7036e4b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7036e4b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7036e4b8

Branch: refs/heads/yang-m1
Commit: 7036e4b8fb594af75ee7127ae8cbbe46ade32514
Parents: 36c7642
Author: Hongbin Ma 
Authored: Mon Apr 11 17:02:52 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 11 17:02:52 2016 +0800

--
 .../kylin/common/util/CacheBuilderTest.java | 43 
 .../cube/v2/filter/MassInValueProviderImpl.java | 40 --
 2 files changed, 79 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7036e4b8/core-common/src/test/java/org/apache/kylin/common/util/CacheBuilderTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/CacheBuilderTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/CacheBuilderTest.java
new file mode 100644
index 000..e30f0b3
--- /dev/null
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/CacheBuilderTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.junit.Test;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
+public class CacheBuilderTest {
+@Test
+public void foo() {
+Cache build = 
CacheBuilder.newBuilder().maximumSize(1).weakValues().removalListener(new 
RemovalListener() {
+@Override
+public void onRemoval(RemovalNotification 
notification) {
+System.out.println(notification.getCause());
+}
+}).build();
+
+build.put(1, 1);
+build.put(1, 2);
+build.put(2, 2);
+
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/7036e4b8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
index 83a6671..16157ec 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/filter/MassInValueProviderImpl.java
@@ -24,10 +24,12 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.metadata.filter.UDF.MassInValueProvider;
 import org.apache.kylin.metadata.filter.function.Functions;
@@ -35,11 +37,22 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Stopwatch;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
 import com.google.common.collect.Sets;
 
 public class MassInValueProviderImpl implements MassInValueProvider {
 public static final Logger logger = 
LoggerFactory.getLogger(MassInValueProviderImpl.class);
 
+private final static Cache> hdfs_caches 
=

kylin git commit: revise advanced settings

2016-04-08 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/document 17a3d91be -> ba80d86f0


revise advanced settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba80d86f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba80d86f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba80d86f

Branch: refs/heads/document
Commit: ba80d86f01ffa03cceb64976adbb7f41bcabdc42
Parents: 17a3d91
Author: Hongbin Ma 
Authored: Fri Apr 8 14:52:22 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Apr 8 14:59:01 2016 +0800

--
 website/_docs15/install/advance_settings.md | 25 
 1 file changed, 12 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ba80d86f/website/_docs15/install/advance_settings.md
--
diff --git a/website/_docs15/install/advance_settings.md 
b/website/_docs15/install/advance_settings.md
index a2da72c..b2f4f04 100644
--- a/website/_docs15/install/advance_settings.md
+++ b/website/_docs15/install/advance_settings.md
@@ -5,9 +5,17 @@ categories: install
 permalink: /docs15/install/advance_settings.html
 ---
 
-## Enable LZO compression
+## Settings for compression
 
-By default Kylin leverages snappy compression to compress the output of MR 
jobs, as well as hbase table storage, reducing the storage overhead. We do not 
choose LZO compression in Kylin because hadoop venders tend to not include LZO 
in their distributions due to license(GPL) issues. To enable LZO in Kylin, 
follow these steps:
+### Disable compression (Easiest way to address compression related issues)
+
+By default Kylin leverages snappy compression to compress the output of MR 
jobs, as well as hbase table storage, to reduce the storage overhead. We do not 
choose LZO compression in Kylin because hadoop venders tend to not include LZO 
in their distributions due to license(GPL) issues. If you compression related 
issues happened in your cubing job, you have two options: 1. Disable 
compression 2. Choose other compression algorithms like LZO. 
+
+ Disable HBase compression
+
+Compression settings only take effect after restarting Kylin server instance 
(by `./kylin.sh start` and `./kylin.sh stop`). To disable compressing MR jobs 
you need to modify $KYLIN_HOME/conf/kylin_job_conf.xml by removing all 
configuration entries related to compression(Just grep the keyword "compress"). 
To disable compressing hbase tables you need to open 
$KYLIN_HOME/conf/kylin.properties and remove the line starting with 
kylin.hbase.default.compression.codec.
+
+### LZO compression
 
  Make sure LZO is working in your environment
 
@@ -25,18 +33,9 @@ To test if the hbase cluster is ready to create LZO 
compressed tables, test foll
 create 'lzoTable', {NAME => 'colFam',COMPRESSION => 'LZO'}
 {% endhighlight %}
 
- Use LZO for HBase compression
-
-You'll need to stop Kylin first by running `./kylin.sh stop`, and then modify 
$KYLIN_HOME/conf/kylin_job_conf.xml by uncommenting some configuration entries 
related to LZO compression. 
-After this, you need to run `./kylin.sh start` to start Kylin again. Now Kylin 
will use LZO to compress MR outputs and hbase tables.
-
-Goto $KYLIN_HOME/conf/kylin.properties, change 
kylin.hbase.default.compression.codec=snappy to 
kylin.hbase.default.compression.codec=lzo
-
- Use LZO for MR jobs
-
-Modify $KYLIN_HOME/conf/kylin_job_conf.xml by changing all 
org.apache.hadoop.io.compress.SnappyCodec to 
com.hadoop.compression.lzo.LzoCodec. 
+ Use LZO compression
 
-Start Kylin again. Now Kylin will use LZO to compress MR outputs and HBase 
tables.
+Compression settings only take effect after restarting Kylin server instance 
(by `./kylin.sh start` and `./kylin.sh stop`). To use LZO for compressing MR 
jobs you need to modify $KYLIN_HOME/conf/kylin_job_conf.xml by replacing 
configuration entries related to compression from 
org.apache.hadoop.io.compress.SnappyCodec to com.hadoop.compression.lzo. You 
can refer to other documents for details: 
http://xiaming.me/posts/2014/05/03/enable-lzo-compression-on-hadoop-pig-and-spark.
  To use LZO for compressing hbase tables you need to open 
$KYLIN_HOME/conf/kylin.properties, change 
kylin.hbase.default.compression.codec=snappy to 
kylin.hbase.default.compression.codec=lzo.
 
 ## Enable LDAP or SSO authentication

kylin git commit: revise log in endpoint rpc client

2016-04-08 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 1e8b44f3f -> 36c76427a


revise log in endpoint rpc client


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/36c76427
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/36c76427
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/36c76427

Branch: refs/heads/yang-m1
Commit: 36c76427a2d795892d1d12ec2770036deb4432bf
Parents: 1e8b44f
Author: Hongbin Ma 
Authored: Fri Apr 8 14:25:27 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Apr 8 14:26:46 2016 +0800

--
 .../storage/hbase/cube/v2/CubeHBaseEndpointRPC.java| 13 -
 1 file changed, 4 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/36c76427/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 31977ae..6bbb0b7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -274,7 +274,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
hbaseColumnsToGTIntList.add(IntList.newBuilder().addAllInts(list).build());
 }
 
-boolean scanLogged = false;
 for (GTScanRequest req : scanRequests) {
 ByteBuffer buffer = 
ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
 GTScanRequest.serializer.serialize(req, buffer);
@@ -290,18 +289,14 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
 logger.debug("Serialized scanRequestBytes {} bytes, 
rawScanBytesString {} bytes", buffer.limit() - buffer.position(), 
rawScanBuffer.limit() - rawScanBuffer.position());
 
-if (!scanLogged) {
-logger.info("The scan(s) info for current segment is as below, 
shard part of start/end key is set to 0", cubeSeg);
-logScan(rawScan, cubeSeg.getStorageLocationIdentifier());
-scanLogged = true;
-}
+logger.info("The scan {} for segment {} is as below, shard part of 
start/end key is set to 0", Integer.toHexString(System.identityHashCode(req)), 
cubeSeg);
+logScan(rawScan, cubeSeg.getStorageLocationIdentifier());
 }
 
 logger.debug("Submitting rpc to {} shards starting from shard {}, scan 
requests count {}", new Object[] { shardNum, cuboidBaseShard, 
scanRequests.size() });
 
 final AtomicInteger totalScannedCount = new AtomicInteger(0);
 final ExpectedSizeIterator epResultItr = new 
ExpectedSizeIterator(scanRequests.size() * shardNum);
-final String currentThreadName = Thread.currentThread().getName();
 
 for (final Pair epRange : 
getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
 executorService.submit(new Runnable() {
@@ -320,12 +315,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 try {
 results = getResults(builder.build(), 
conn.getTable(cubeSeg.getStorageLocationIdentifier()), epRange.getFirst(), 
epRange.getSecond());
 } catch (Throwable throwable) {
-throw new RuntimeException("Error when visiting 
cubes by endpoint:", throwable);
+throw new RuntimeException(" " + 
"Error when visiting cubes by endpoint", throwable);
 }
 
 for (Map.Entry result : results.entrySet()) {
 
totalScannedCount.addAndGet(result.getValue().getStats().getScannedRowCount());
-logger.info(" " + getStatsString(result));
+logger.info(" " + 
getStatsString(result));
 try {
 
epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows(;
 } catch (IOException | DataFormatException e) {

kylin git commit: minor bug fix in project L2 cache rebuilding

2016-04-07 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang-m1 143dfb5a7 -> 1e8b44f3f


minor bug fix in project L2 cache rebuilding


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e8b44f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e8b44f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e8b44f3

Branch: refs/heads/yang-m1
Commit: 1e8b44f3ff83c457f51732bd9579a0d1638a67da
Parents: 143dfb5
Author: Hongbin Ma 
Authored: Fri Apr 8 11:27:20 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Apr 8 13:30:24 2016 +0800

--
 .../src/main/java/org/apache/kylin/metadata/MetadataManager.java | 4 
 .../apache/kylin/rest/controller/ExternalFilterController.java   | 1 +
 .../main/java/org/apache/kylin/rest/service/CacheService.java| 2 +-
 .../java/org/apache/kylin/rest/service/ExtFilterService.java | 2 --
 4 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e8b44f3/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 928baad..9ed1f13 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -351,6 +351,10 @@ public class MetadataManager {
 return t;
 }
 
+public void reloadExtFilter(String extFilterName) throws IOException {
+
reloadExternalFilterAt(ExternalFilterDesc.concatResourcePath(extFilterName));
+}
+
 public void reloadSourceTableExt(String tableIdentity) throws IOException {
 reloadSourceTableExdAt(TableDesc.concatExdResourcePath(tableIdentity));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e8b44f3/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
 
b/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
index cc56059..0d41847 100644
--- 
a/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
+++ 
b/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
@@ -71,6 +71,7 @@ public class ExternalFilterController extends BasicController 
{
 Map result = new HashMap();
 ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), 
ExternalFilterDesc.class);
 extFilterService.updateExternalFilter(desc);
+extFilterService.syncExtFilterToProject(new String[] { desc.getName() 
}, request.getProject());
 result.put("success", "true");
 return result;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e8b44f3/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 4ec2116..48afde6 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -201,7 +201,7 @@ public class CacheService extends BasicService {
 CubeDescManager.clearCache();
 break;
 case EXTERNAL_FILTER:
-getMetadataManager().reloadTableCache(cacheKey);
+getMetadataManager().reloadExtFilter(cacheKey);
 IIDescManager.clearCache();
 CubeDescManager.clearCache();
 break;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e8b44f3/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java 
b/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
index 7e72d9a..576ece9 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
@@ -62,8 +62,6 @@ public class ExtFilterService extends BasicService {
 getMetadataManager().removeExternalFilter(name);
 }
 
-
-
 @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
 public void syncExtFilterToProject(String[] filters, String project) 
throws

kylin git commit: put ext filter into project L2 cache

2016-04-01 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master c86de1a8e -> 0e662a350


put ext filter into project L2 cache


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0e662a35
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0e662a35
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0e662a35

Branch: refs/heads/master
Commit: 0e662a350b88ad4b1194874af19f25590286a52a
Parents: c86de1a
Author: Hongbin Ma 
Authored: Fri Apr 1 15:47:50 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Apr 1 15:47:50 2016 +0800

--
 .../metadata/filter/UDF/MassInTupleFilter.java  |  1 +
 .../kylin/metadata/project/ProjectL2Cache.java  | 37 +++-
 .../kylin/metadata/project/ProjectManager.java  |  9 +
 .../controller/ExternalFilterController.java| 37 ++--
 .../kylin/rest/service/ExtFilterService.java| 21 ---
 5 files changed, 58 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 0cb416c..5889c61 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.function.Functions;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index e8004e1..3006773 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -59,6 +60,16 @@ class ProjectL2Cache {
 projectCaches.clear();
 }
 
+public ExternalFilterDesc getExternalFilterDesc(String project, String 
extFilterName) {
+ProjectCache prjCache = getCache(project);
+return prjCache.extFilters.get(extFilterName);
+}
+
+public Map listExternalFilterDesc(String 
project) {
+ProjectCache prjCache = getCache(project);
+return Collections.unmodifiableMap(prjCache.extFilters);
+}
+
 public List listDefinedTables(String project) {
 ProjectCache prjCache = getCache(project);
 List result = 
Lists.newArrayListWithCapacity(prjCache.tables.size());
@@ -159,7 +170,7 @@ class ProjectL2Cache {
 
 private ProjectCache loadCache(String project) {
 logger.info("Loading L2 project cache for " + project);
-ProjectCache result = new ProjectCache(project);
+ProjectCache projectCache = new ProjectCache(project);
 
 ProjectInstance pi = mgr.getProject(project);
 
@@ -171,30 +182,39 @@ class ProjectL2Cache {
 for (String tableName : pi.getTables()) {
 TableDesc tableDesc = metaMgr.getTableDesc(tableName);
 if (tableDesc != null) {
-result.tables.put(tableDesc.getIdentity(), new 
TableCache(tableDesc));
+projectCache.tables.put(tableDesc.getIdentity(), new 
TableCache(tableDesc));
 } else {
 logger.warn("Table '" + tableName + "' defined under project 
'" + project + "' is not found");
 }
 }
 
+for (String extFilterName : pi.getExtFilters()) {
+ExternalFilterDesc filterDesc = 
metaMgr.getExtFilterDesc(extFilterName);
+if (filterDesc != null) {
+

[1/4] kylin git commit: KYLIN-1556 minor changes to improve correctness, still needs further changes

2016-04-01 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/master 195f75104 -> c86de1a8e


KYLIN-1556 minor changes to improve correctness, still needs further changes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9aeb7652
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9aeb7652
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9aeb7652

Branch: refs/heads/master
Commit: 9aeb76526f71a43ebfd9394c1eafb46398906208
Parents: 5dd133f
Author: Hongbin Ma 
Authored: Fri Apr 1 14:45:02 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Apr 1 14:45:57 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 16 ++--
 .../org/apache/kylin/cube/CubeInstance.java |  9 +++--
 .../org/apache/kylin/query/ITIIQueryTest.java   |  2 +-
 .../apache/kylin/query/routing/Candidate.java   | 20 +++-
 4 files changed, 25 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9aeb7652/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3d9a338..e911dd9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -66,11 +66,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 // 

 
 private volatile Properties properties = new Properties();
-
+
 public KylinConfigBase() {
 this(new Properties());
 }
-
+
 public KylinConfigBase(Properties props) {
 this.properties = props;
 }
@@ -433,7 +433,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public int getScanThreshold() {
 return Integer.parseInt(getOptional("kylin.query.scan.threshold", 
"1000"));
 }
-
+
 public int getCubeVisitTimeoutTimes() {
 return 
Integer.parseInt(getOptional("kylin.query.cube.visit.timeout.times", "1"));
 }
@@ -447,7 +447,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getBadQueryDefaultAlertingSeconds() {
-return 
Integer.parseInt(getOptional("kylin.query.badquery.default.alerting.seconds", 
"90"));
+return 
Integer.parseInt(getOptional("kylin.query.badquery.alerting.seconds", "90"));
+}
+
+public int getBadQueryDefaultDetectIntervalSeconds() {
+return 
Integer.parseInt(getOptional("kylin.query.badquery.default.alerting.seconds", 
"60"));
 }
 
 public int getCachedDictMaxEntrySize() {
@@ -500,7 +504,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 //don't change this per https://issues.apache.org/jira/browse/KYLIN-1545
 public int getHBaseScanMaxResultSize() {
-return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "-1")); 
+return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "-1"));
 }
 
 public int getCubingInMemSamplingPercent() {
@@ -532,7 +536,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public String getHbaseDefaultCompressionCodec() {
 return getOptional("kylin.hbase.default.compression.codec", "");
 }
-
+
 public String getHbaseDefaultEncoding() {
 return getOptional("kylin.hbase.default.encoding", "FAST_DIFF");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9aeb7652/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index f725308..8f1af7a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -343,7 +343,8 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 private int getCost(SQLDigest digest) {
 int calculatedCost = cost;
 
-calculatedCost += getAllDimensions().size() * COST_WEIGHT_DIMENSION + 
getMeasures().size() * COST_WEIGHT_MEASURE;
+//the number of dimensions is not as accurate as number of row key cols
+calculatedCost += getRowKeyColumnCount() * COST_WEIGHT_DIMENSION + 
getMeasures().size() * COST_WEIGHT_MEASURE;
 
 for (LookupDesc lookupDesc : 
this.getDescriptor().getModel().getLookups()) {
 // more

[2/4] kylin git commit: KYLIN-1308 allow to multiply cube visit timeout by a factor

2016-04-01 Thread mahongbin

KYLIN-1308 allow to multiply cube visit timeout by a factor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5dd133f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5dd133f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5dd133f6

Branch: refs/heads/master
Commit: 5dd133f6d05b8e8c65c67d557966ebcdd88a6093
Parents: 27f7917
Author: Hongbin Ma 
Authored: Fri Apr 1 11:15:37 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Apr 1 14:45:57 2016 +0800

--
 .../org/apache/kylin/common/KylinConfigBase.java| 16 
 .../storage/hbase/cube/v2/CubeHBaseEndpointRPC.java |  2 ++
 2 files changed, 10 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5dd133f6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e97051b..3d9a338 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -399,13 +399,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 String cut;
 switch (capacity) {
 case "SMALL":
-cut = getOptional("kylin.hbase.region.cut.small", "5");
+cut = getOptional("kylin.hbase.region.cut.small", "10");
 break;
 case "MEDIUM":
-cut = getOptional("kylin.hbase.region.cut.medium", "10");
+cut = getOptional("kylin.hbase.region.cut.medium", "20");
 break;
 case "LARGE":
-cut = getOptional("kylin.hbase.region.cut.large", "50");
+cut = getOptional("kylin.hbase.region.cut.large", "100");
 break;
 default:
 throw new IllegalArgumentException("Capacity not recognized: " + 
capacity);
@@ -433,6 +433,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 public int getScanThreshold() {
 return Integer.parseInt(getOptional("kylin.query.scan.threshold", 
"1000"));
 }
+
+public int getCubeVisitTimeoutTimes() {
+return 
Integer.parseInt(getOptional("kylin.query.cube.visit.timeout.times", "1"));
+}
 
 public int getBadQueryStackTraceDepth() {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
@@ -443,11 +447,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getBadQueryDefaultAlertingSeconds() {
-return 
Integer.parseInt(getOptional("kylin.query.badquery.alerting.seconds", "90"));
-}
-
-public int getBadQueryDefaultDetectIntervalSeconds() {
-return 
Integer.parseInt(getOptional("kylin.query.badquery.detect.interval.seconds", 
"60"));
+return 
Integer.parseInt(getOptional("kylin.query.badquery.default.alerting.seconds", 
"90"));
 }
 
 public int getCachedDictMaxEntrySize() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/5dd133f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 2138bcc..f36ab29 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
 import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.BytesSerializer;
@@ -120,6 +121,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 this.expectedSize = expectedSize;
 this.queue = new ArrayBlockingQueue(expectedSize);
 this.timeout = 
HadoopUtil.getCurrentConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 
HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+this.timeout *= 
KylinConfig.getInstanceFromEnv().getCubeVisitTimeoutTimes();
 logger.info("Timeout for ExpectedSizeIterator is " + this.timeout);
 }

< 2 3 4 5 6 7 8 9 10 11 >

601 - 700 of 1363 matches

Mail list logo