[09/13] kylin git commit: Revert "rename the streaming_table.json"

2016-09-19 Thread mahongbin 
Revert "rename the streaming_table.json"

This reverts commit ffdc5d21d5f46daaf25a0dfed9b3afeef78fc62c.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bec25b46
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bec25b46
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bec25b46

Branch: refs/heads/master
Commit: bec25b4638835301a102141e56e12cd38d09139f
Parents: d1e979b
Author: Hongbin Ma 
Authored: Mon Sep 19 23:50:13 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Sep 20 11:43:08 2016 +0800

--
 .../kafka/DEFAULT.STREAMING_TABLE.json  | 21 
 .../streaming/DEFAULT.STREAMING_TABLE.json  |  6 ++
 2 files changed, 27 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bec25b46/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
new file mode 100644
index 000..6a64cce
--- /dev/null
+++ b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
@@ -0,0 +1,21 @@
+{
+ 
+  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
+  "name": "DEFAULT.STREAMING_TABLE",
+  "topic": "test_streaming_table_topic_xyz",
+  "timeout": 6,
+  "bufferSize": 65536,
+  "parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
+  "last_modified": 0,
+  "clusters": [
+{
+  "brokers": [
+{
+  "id": 0,
+  "host": "sandbox",
+  "port": 6667
+}
+  ]
+}
+  ]
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/bec25b46/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
new file mode 100644
index 000..85a477b
--- /dev/null
+++ b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
@@ -0,0 +1,6 @@
+{
+  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
+  "name": "DEFAULT.STREAMING_TABLE",
+  "type": "kafka",
+  "last_modified": 0
+}

[13/13] kylin git commit: Revert "KYLIN-1726 update to kafka 0.10"

2016-09-19 Thread mahongbin 
Revert "KYLIN-1726 update to kafka 0.10"

This reverts commit 1b1b2e37fdcba7ad67f0fa3f2369aa65431f13bc.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/355e58ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/355e58ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/355e58ba

Branch: refs/heads/master
Commit: 355e58ba4209ddf945663228688d550bf654c387
Parents: da5ba27
Author: Hongbin Ma 
Authored: Mon Sep 19 23:50:26 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Sep 20 11:43:08 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 30 
 .../kylin/rest/controller/CubeController.java   |  8 ++---
 .../apache/kylin/rest/service/JobService.java   |  4 +--
 .../source/kafka/util/KafkaSampleProducer.java  | 38 
 4 files changed, 27 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/355e58ba/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 11eabce..fc68798 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -679,28 +679,12 @@ public class CubeManager implements IRealizationProvider {
 return null;
 }
 
-List buildingSegs = cube.getBuildingSegments();
-if (buildingSegs.size() > 0) {
-logger.debug("Cube " + cube.getName() + " has " + 
buildingSegs.size() + " building segments");
-}
-
-List readySegs = 
cube.getSegments(SegmentStatusEnum.READY);
-
-List mergingSegs = Lists.newArrayList();
-if (buildingSegs.size() > 0) {
-
-for (CubeSegment building : buildingSegs) {
-// exclude those under-merging segs
-for (CubeSegment ready : readySegs) {
-if (ready.getSourceOffsetStart() >= 
building.getSourceOffsetStart() && ready.getSourceOffsetEnd() <= 
building.getSourceOffsetEnd()) {
-mergingSegs.add(ready);
-}
-}
-}
+if (cube.getBuildingSegments().size() > 0) {
+logger.debug("Cube " + cube.getName() + " has bulding segment, 
will not trigger merge at this moment");
+return null;
 }
 
-// exclude those already under merging segments
-readySegs.removeAll(mergingSegs);
+List ready = cube.getSegments(SegmentStatusEnum.READY);
 
 long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges();
 Arrays.sort(timeRanges);
@@ -708,9 +692,9 @@ public class CubeManager implements IRealizationProvider {
 for (int i = timeRanges.length - 1; i >= 0; i--) {
 long toMergeRange = timeRanges[i];
 
-for (int s = 0; s < readySegs.size(); s++) {
-CubeSegment seg = readySegs.get(s);
-Pair p = 
findMergeOffsetsByDateRange(readySegs.subList(s, readySegs.size()), //
+for (int s = 0; s < ready.size(); s++) {
+CubeSegment seg = ready.get(s);
+Pair p = 
findMergeOffsetsByDateRange(ready.subList(s, ready.size()), //
 seg.getDateRangeStart(), seg.getDateRangeStart() + 
toMergeRange, toMergeRange);
 if (p != null && p.getSecond().getDateRangeEnd() - 
p.getFirst().getDateRangeStart() >= toMergeRange)
 return Pair.newPair(p.getFirst().getSourceOffsetStart(), 
p.getSecond().getSourceOffsetEnd());

http://git-wip-us.apache.org/repos/asf/kylin/blob/355e58ba/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 669f53e..42b117c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -272,7 +272,7 @@ public class CubeController extends BasicController {
 @RequestMapping(value = "/{cubeName}/rebuild", method = { 
RequestMethod.PUT })
 @ResponseBody
 public JobInstance rebuild(@PathVariable String cubeName, @RequestBody 
JobBuildRequest req) {
-return buildInternal(cubeName, req.getStartTime(), req.getEndTime(), 
0, 0, req.getBuildType(), true, req.isForce()

[01/13] kylin git commit: Revert "KYLIN-1726 Scalable streaming cubing"

2016-09-19 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 20ac92a32 -> 506cd7831


http://git-wip-us.apache.org/repos/asf/kylin/blob/506cd783/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index ab8b161..20c57a9 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -1,20 +1,37 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
+ *
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ *  contributor license agreements. See the NOTICE file distributed with
+ *
+ *  this work for additional information regarding copyright ownership.
+ *
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ *  (the "License"); you may not use this file except in compliance with
+ *
+ *  the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ *  See the License for the specific language governing permissions and
+ *
+ *  limitations under the License.
+ *
+ * /
+ */
+
 package org.apache.kylin.source.kafka;
 
 import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/kylin/blob/506cd783/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java
index d84d3db..4145ef6 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TopicMeta.java
@@ -1,20 +1,37 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
+ *
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ *  contributor license agreements. See the NOTICE file distributed with
+ *
+ *  this work for additional information regarding copyright ownership.
+ *
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ *  (the "License"); you may not use this file except in compliance with
+ *
+ *  the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ *  See the License for the specific language governing permissions and
+ *
+ *  limitations under the License.
+ *
+ * /
+ */
+
 package org.apache.kylin.source.kafka;
 
 import java.util.Collections;

http://git-wip-us.apache.org/repos/asf/kylin/blob/506cd783/source-kafka/src/main/java/org/apache/kylin/source/kafka/UpdateTimeRangeStep.java

[11/13] kylin git commit: Revert "KYLIN-1762 discard job when no stream message"

2016-09-19 Thread mahongbin 
Revert "KYLIN-1762 discard job when no stream message"

This reverts commit 1108d9eeccecbccffea0b3f9049151672196c91a.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da5ba276
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da5ba276
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da5ba276

Branch: refs/heads/master
Commit: da5ba276b972f8b3c0d220252e74ac2ff73298fc
Parents: bec25b4
Author: Hongbin Ma 
Authored: Mon Sep 19 23:50:20 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Sep 20 11:43:08 2016 +0800

--
 .../job/execution/DefaultChainedExecutable.java |  6 ---
 .../kylin/source/kafka/SeekOffsetStep.java  | 45 +---
 2 files changed, 10 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/da5ba276/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 39a5f4f..753b389 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -88,7 +88,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 boolean allSucceed = true;
 boolean hasError = false;
 boolean hasRunning = false;
-boolean hasDiscarded = false;
 for (Executable task : jobs) {
 final ExecutableState status = task.getStatus();
 if (status == ExecutableState.ERROR) {
@@ -100,9 +99,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 if (status == ExecutableState.RUNNING) {
 hasRunning = true;
 }
-if (status == ExecutableState.DISCARDED) {
-hasDiscarded = true;
-}
 }
 if (allSucceed) {
 setEndTime(System.currentTimeMillis());
@@ -114,8 +110,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 notifyUserStatusChange(executableContext, 
ExecutableState.ERROR);
 } else if (hasRunning) {
 jobService.updateJobOutput(getId(), ExecutableState.RUNNING, 
null, null);
-} else if (hasDiscarded) {
-jobService.updateJobOutput(getId(), ExecutableState.DISCARDED, 
null, null);
 } else {
 jobService.updateJobOutput(getId(), ExecutableState.READY, 
null, null);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/da5ba276/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
index 479f1b8..5dca93f 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
@@ -17,10 +17,6 @@
 */
 package org.apache.kylin.source.kafka;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Maps;
-import org.apache.commons.math3.util.MathUtils;
 import org.apache.kylin.source.kafka.util.KafkaClient;
 import org.apache.kylin.source.kafka.util.KafkaOffsetMapping;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -38,7 +34,6 @@ import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -106,39 +101,19 @@ public class SeekOffsetStep extends AbstractExecutable {
 }
 }
 
-long totalStartOffset = 0, totalEndOffset = 0;
-for (Long v : startOffsets.values()) {
-totalStartOffset += v;
-}
-for (Long v : endOffsets.values()) {
-totalEndOffset += v;
-}
+KafkaOffsetMapping.saveOffsetStart(segment, startOffsets);
+KafkaOffsetMapping.saveOffsetEnd(segment, endOffsets);
 
-if (totalEndOffset > totalStartOffset) {
-KafkaOffsetMapping.saveOffsetStart(segment, startOffsets);
-KafkaOffsetMapping.saveOffsetEnd(segment, endOffsets);
-

[02/13] kylin git commit: Revert "KYLIN-1726 Scalable streaming cubing"

2016-09-19 Thread mahongbin 
Revert "KYLIN-1726 Scalable streaming cubing"

This reverts commit 81c7323b633df88eedac8b319fc57f9b62b01a4a.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/506cd783
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/506cd783
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/506cd783

Branch: refs/heads/master
Commit: 506cd783132023a06f1669ad248b74bf9d96d0e1
Parents: 1f48804
Author: Hongbin Ma 
Authored: Mon Sep 19 23:55:54 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Sep 20 11:43:08 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  35 +--
 .../kylin/job/streaming/Kafka10DataLoader.java  |  80 ---
 .../apache/kylin/common/KylinConfigBase.java|   1 -
 .../java/org/apache/kylin/cube/CubeSegment.java |   1 -
 .../java/org/apache/kylin/cube/ISegment.java|  39 
 .../cube/gridtable/SegmentGTStartAndEnd.java|   2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |   6 -
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   6 -
 .../apache/kylin/gridtable/ScannerWorker.java   |   2 +-
 .../metadata/model/IJoinedFlatTableDesc.java|   2 -
 .../apache/kylin/metadata/model/ISegment.java   |  36 ---
 .../kylin/engine/mr/BatchMergeJobBuilder2.java  |   3 -
 .../org/apache/kylin/engine/mr/IMRInput.java|  10 -
 .../java/org/apache/kylin/engine/mr/MRUtil.java |   4 -
 .../test_streaming_table_model_desc.json|   6 +-
 .../kylin/provision/BuildCubeWithStream.java| 218 +-
 .../org/apache/kylin/provision/MockKafka.java   | 191 
 .../apache/kylin/provision/NetworkUtils.java|  52 -
 pom.xml |   2 +-
 .../apache/kylin/source/hive/HiveMRInput.java   |  11 -
 source-kafka/pom.xml|  13 +-
 .../kylin/source/kafka/KafkaConfigManager.java  |  46 ++--
 .../apache/kylin/source/kafka/KafkaMRInput.java | 221 ---
 .../apache/kylin/source/kafka/KafkaSource.java  |  57 -
 .../kylin/source/kafka/KafkaStreamingInput.java |  65 +++---
 .../kylin/source/kafka/MergeOffsetStep.java |  89 
 .../kylin/source/kafka/SeekOffsetStep.java  | 119 --
 .../kylin/source/kafka/StreamingParser.java |  49 ++--
 .../source/kafka/StringStreamingParser.java |  49 ++--
 .../source/kafka/TimedJsonStreamParser.java |  49 ++--
 .../apache/kylin/source/kafka/TopicMeta.java|  49 ++--
 .../kylin/source/kafka/UpdateTimeRangeStep.java | 108 -
 .../source/kafka/config/KafkaClusterConfig.java |   3 +-
 .../source/kafka/hadoop/KafkaFlatTableJob.java  | 165 --
 .../kafka/hadoop/KafkaFlatTableMapper.java  |  51 -
 .../source/kafka/hadoop/KafkaInputFormat.java   |  98 
 .../kafka/hadoop/KafkaInputRecordReader.java| 166 --
 .../source/kafka/hadoop/KafkaInputSplit.java| 102 -
 .../kylin/source/kafka/util/KafkaClient.java| 115 --
 .../source/kafka/util/KafkaOffsetMapping.java   |  97 
 .../kylin/source/kafka/util/KafkaRequester.java |  56 +++--
 .../kylin/source/kafka/util/KafkaUtils.java |   3 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   2 +-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |   2 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   2 +-
 45 files changed, 348 insertions(+), 2135 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/506cd783/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 9b282e3..8c64f91 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -143,12 +143,14 @@ public class DeployUtil {
 deployHiveTables();
 }
 
-public static void prepareTestDataForStreamingCube(long startTime, long 
endTime, int numberOfRecords, String cubeName, StreamDataLoader 
streamDataLoader) throws IOException {
+public static void prepareTestDataForStreamingCube(long startTime, long 
endTime, String cubeName, StreamDataLoader streamDataLoader) throws IOException 
{
 CubeInstance cubeInstance = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
-List data = 
StreamingTableDataGenerator.generate(numberOfRecords, startTime, endTime, 
cubeInstance.getFactTable());
+List data = StreamingTableDataGenerator.generate(1, 
startTime, endTime, cubeInstance.getFactTable());
+List data2 = StreamingTableDataGenerator.generate(10, endTime, 
endTime + 30, cubeInstance.getFactTable());

kylin git commit: update KylinVersion to 1.6.0

2016-09-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master e17b11bd8 -> 20ac92a32


update KylinVersion to 1.6.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20ac92a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20ac92a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20ac92a3

Branch: refs/heads/master
Commit: 20ac92a325ef4e92839c18b5e9c8e2bfcaf56c37
Parents: e17b11b
Author: Li Yang 
Authored: Tue Sep 20 08:38:49 2016 +0800
Committer: Li Yang 
Committed: Tue Sep 20 08:38:49 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinVersion.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/20ac92a3/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 87a44c3..6aa773c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -67,7 +67,7 @@ public class KylinVersion {
 /**
  * Require MANUAL updating kylin version per ANY upgrading.
  */
-private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("1.5.4");
+private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("1.6.0");
 
 private static final Set SIGNATURE_INCOMPATIBLE_REVISIONS = 
new HashSet();

[04/13] kylin git commit: Revert "KYLIN-1726 add test case BuildCubeWithStream2"

2016-09-19 Thread mahongbin 
Revert "KYLIN-1726 add test case BuildCubeWithStream2"

This reverts commit 3e081b3fbec4fc8a6cc4ddf8795d2fd581ae04f4.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f9692fa5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f9692fa5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f9692fa5

Branch: refs/heads/tempmaster
Commit: f9692fa5599de04030599b699f5460bdc7cf0e42
Parents: c68bba0
Author: Hongbin Ma 
Authored: Mon Sep 19 23:49:51 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 23:49:51 2016 +0800

--
 .../kylin/job/streaming/Kafka10DataLoader.java  |   4 +
 .../apache/kylin/common/KylinConfigBase.java|   4 -
 .../java/org/apache/kylin/cube/CubeManager.java |  28 +-
 .../org/apache/kylin/job/dao/ExecutableDao.java |   1 -
 .../kylin/job/manager/ExecutableManager.java|   2 +-
 .../streaming/cube/StreamingCubeBuilder.java|   2 +-
 .../test_streaming_table_cube_desc.json |   3 +-
 .../kylin/provision/BuildCubeWithStream.java|  32 +--
 .../kylin/provision/BuildCubeWithStream2.java   | 274 ---
 .../kylin/rest/controller/CubeController.java   |   8 +-
 .../apache/kylin/rest/service/JobService.java   |   4 +-
 .../kylin/source/kafka/SeekOffsetStep.java  |   7 +-
 12 files changed, 49 insertions(+), 320 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f9692fa5/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java 
b/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java
index 2b299cc..a5132af 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java
@@ -65,9 +65,13 @@ public class Kafka10DataLoader extends StreamDataLoader {
 props.put("retry.backoff.ms", "1000");
 KafkaProducer producer = KafkaClient.getKafkaProducer(brokerList, 
props);
 
+int boundary = messages.size() / 10;
 for (int i = 0; i < messages.size(); ++i) {
 ProducerRecord keyedMessage = new 
ProducerRecord(clusterConfig.getTopic(), String.valueOf(i), 
messages.get(i));
 producer.send(keyedMessage);
+if (i % boundary == 0) {
+logger.info("sending " + i + " messages to " + 
this.toString());
+}
 }
 logger.info("sent " + messages.size() + " messages to " + 
this.toString());
 producer.close();

http://git-wip-us.apache.org/repos/asf/kylin/blob/f9692fa5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3b06ed8..fafb1fc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -809,8 +809,4 @@ abstract public class KylinConfigBase implements 
Serializable {
 public String getCreateFlatHiveTableMethod() {
 return getOptional("kylin.hive.create.flat.table.method", "1");
 }
-
-public int getMaxBuildingSegments() {
-return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"1"));
-}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f9692fa5/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 57b9510..d494fcc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -400,8 +400,13 @@ public class CubeManager implements IRealizationProvider {
 }
 
 public CubeSegment appendSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset) throws IOException {
+return appendSegment(cube, startDate, endDate, startOffset, endOffset, 
true);
+}
+
+public CubeSegment appendSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset, boolean strictChecking) throws 
IOException {
 
-checkBuildingSegment(cube);
+if (strictChecking)
+checkNoBuildingSegment(cube);
 
 if

[06/13] kylin git commit: Revert "change to upper case"

2016-09-19 Thread mahongbin 
Revert "change to upper case"

This reverts commit aa30880578078369a5844e04a7d7ce736661e902.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a70108e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a70108e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a70108e5

Branch: refs/heads/tempmaster
Commit: a70108e52f9570aa29b4f811316c5e68c20b2f15
Parents: 1093f34
Author: Hongbin Ma 
Authored: Mon Sep 19 23:50:04 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 23:50:04 2016 +0800

--
 .../kafka/DEFAULT.STREAMING_TABLE.json  | 21 
 .../streaming/DEFAULT.STREAMING_TABLE.json  |  6 --
 2 files changed, 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a70108e5/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
deleted file mode 100644
index 6a64cce..000
--- a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- 
-  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
-  "name": "DEFAULT.STREAMING_TABLE",
-  "topic": "test_streaming_table_topic_xyz",
-  "timeout": 6,
-  "bufferSize": 65536,
-  "parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
-  "last_modified": 0,
-  "clusters": [
-{
-  "brokers": [
-{
-  "id": 0,
-  "host": "sandbox",
-  "port": 6667
-}
-  ]
-}
-  ]
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/a70108e5/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
deleted file mode 100644
index 85a477b..000
--- a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
-  "name": "DEFAULT.STREAMING_TABLE",
-  "type": "kafka",
-  "last_modified": 0
-}

[09/13] kylin git commit: Revert "KYLIN-1726 update to kafka 0.10"

2016-09-19 Thread mahongbin 
Revert "KYLIN-1726 update to kafka 0.10"

This reverts commit 1b1b2e37fdcba7ad67f0fa3f2369aa65431f13bc.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61ac0964
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61ac0964
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61ac0964

Branch: refs/heads/tempmaster
Commit: 61ac0964606a23ba891fcc79b9d660b4b8976f72
Parents: 4de2c0c
Author: Hongbin Ma 
Authored: Mon Sep 19 23:50:26 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 23:50:26 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 30 
 .../kylin/rest/controller/CubeController.java   |  8 ++---
 .../apache/kylin/rest/service/JobService.java   |  4 +--
 .../source/kafka/util/KafkaSampleProducer.java  | 38 
 4 files changed, 27 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61ac0964/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 11eabce..fc68798 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -679,28 +679,12 @@ public class CubeManager implements IRealizationProvider {
 return null;
 }
 
-List buildingSegs = cube.getBuildingSegments();
-if (buildingSegs.size() > 0) {
-logger.debug("Cube " + cube.getName() + " has " + 
buildingSegs.size() + " building segments");
-}
-
-List readySegs = 
cube.getSegments(SegmentStatusEnum.READY);
-
-List mergingSegs = Lists.newArrayList();
-if (buildingSegs.size() > 0) {
-
-for (CubeSegment building : buildingSegs) {
-// exclude those under-merging segs
-for (CubeSegment ready : readySegs) {
-if (ready.getSourceOffsetStart() >= 
building.getSourceOffsetStart() && ready.getSourceOffsetEnd() <= 
building.getSourceOffsetEnd()) {
-mergingSegs.add(ready);
-}
-}
-}
+if (cube.getBuildingSegments().size() > 0) {
+logger.debug("Cube " + cube.getName() + " has bulding segment, 
will not trigger merge at this moment");
+return null;
 }
 
-// exclude those already under merging segments
-readySegs.removeAll(mergingSegs);
+List ready = cube.getSegments(SegmentStatusEnum.READY);
 
 long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges();
 Arrays.sort(timeRanges);
@@ -708,9 +692,9 @@ public class CubeManager implements IRealizationProvider {
 for (int i = timeRanges.length - 1; i >= 0; i--) {
 long toMergeRange = timeRanges[i];
 
-for (int s = 0; s < readySegs.size(); s++) {
-CubeSegment seg = readySegs.get(s);
-Pair p = 
findMergeOffsetsByDateRange(readySegs.subList(s, readySegs.size()), //
+for (int s = 0; s < ready.size(); s++) {
+CubeSegment seg = ready.get(s);
+Pair p = 
findMergeOffsetsByDateRange(ready.subList(s, ready.size()), //
 seg.getDateRangeStart(), seg.getDateRangeStart() + 
toMergeRange, toMergeRange);
 if (p != null && p.getSecond().getDateRangeEnd() - 
p.getFirst().getDateRangeStart() >= toMergeRange)
 return Pair.newPair(p.getFirst().getSourceOffsetStart(), 
p.getSecond().getSourceOffsetEnd());

http://git-wip-us.apache.org/repos/asf/kylin/blob/61ac0964/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 669f53e..42b117c 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -272,7 +272,7 @@ public class CubeController extends BasicController {
 @RequestMapping(value = "/{cubeName}/rebuild", method = { 
RequestMethod.PUT })
 @ResponseBody
 public JobInstance rebuild(@PathVariable String cubeName, @RequestBody 
JobBuildRequest req) {
-return buildInternal(cubeName, req.getStartTime(), req.getEndTime(), 
0, 0, req.getBuildType(), true,

[11/13] kylin git commit: Revert "KYLIN-1726 use segment uuid instead of name"

2016-09-19 Thread mahongbin 
Revert "KYLIN-1726 use segment uuid instead of name"

This reverts commit 42dafc15db40731582d6257c618eff29643930a8.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc27d52f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc27d52f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc27d52f

Branch: refs/heads/tempmaster
Commit: dc27d52fd676be666ea2778c11b58be05d3cfd6d
Parents: 3f28027
Author: Hongbin Ma 
Authored: Mon Sep 19 23:51:57 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 23:51:57 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  5 +---
 .../kylin/provision/BuildCubeWithStream.java| 26 +++-
 .../apache/kylin/source/kafka/KafkaMRInput.java |  2 +-
 .../source/kafka/hadoop/KafkaFlatTableJob.java  | 11 ++---
 .../kafka/hadoop/KafkaInputRecordReader.java|  9 +++
 5 files changed, 17 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dc27d52f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index fc68798..daeca0d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -444,11 +444,8 @@ public class CubeManager implements IRealizationProvider {
 updateCube(cubeBuilder);
 return newSegment;
 }
-public CubeSegment refreshSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset) throws IOException {
-return refreshSegment(cube, startDate, endDate, startOffset, 
endOffset, true);
-}
 
-public CubeSegment refreshSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset, boolean strictChecking) throws 
IOException {
+public CubeSegment refreshSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset) throws IOException {
 checkNoBuildingSegment(cube);
 
 CubeSegment newSegment = newSegment(cube, startDate, endDate, 
startOffset, endOffset);

http://git-wip-us.apache.org/repos/asf/kylin/blob/dc27d52f/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 9e779ab..7f79acc 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -21,8 +21,6 @@ package org.apache.kylin.provision;
 import java.io.File;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
-import java.util.HashMap;
-import java.util.List;
 import java.util.TimeZone;
 import java.util.UUID;
 
@@ -147,34 +145,18 @@ public class BuildCubeWithStream {
 //merge
 mergeSegment(cubeName, 0, 15000);
 
-List segments = 
cubeManager.getCube(cubeName).getSegments();
-Assert.assertTrue(segments.size() == 1);
-
-CubeSegment toRefreshSeg = segments.get(0);
-HashMap partitionOffsetMap = 
toRefreshSeg.getAdditionalInfo();
-
-refreshSegment(cubeName, toRefreshSeg.getSourceOffsetStart(), 
toRefreshSeg.getSourceOffsetEnd(), partitionOffsetMap);
-segments = cubeManager.getCube(cubeName).getSegments();
-Assert.assertTrue(segments.size() == 1);
-
 }
 
 private String mergeSegment(String cubeName, long startOffset, long 
endOffset) throws Exception {
-CubeSegment segment = 
cubeManager.mergeSegments(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset, false);
+CubeSegment segment = 
cubeManager.mergeSegments(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset, true);
 DefaultChainedExecutable job = 
EngineFactory.createBatchMergeJob(segment, "TEST");
 jobService.addJob(job);
 waitForJob(job.getId());
 return job.getId();
 }
 
-private String refreshSegment(String cubeName, long startOffset, long 
endOffset, HashMap partitionOffsetMap) throws Exception {
-CubeSegment segment = 
cubeManager.refreshSegment(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset, false);
-segment.setAdditionalInfo(partitionOffsetMap);
-CubeInstance cubeInstance = cubeManager.getCube(cubeName);
-CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance);
-

[03/13] kylin git commit: Revert "stop scheduler on test finish"

2016-09-19 Thread mahongbin 
Revert "stop scheduler on test finish"

This reverts commit 023b5cb1dfdacfa9adcc2586b4aaeb3886f61098.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c68bba0e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c68bba0e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c68bba0e

Branch: refs/heads/tempmaster
Commit: c68bba0e27984be5ac6e78945dbef8c46c6d2a61
Parents: 34970d9
Author: Hongbin Ma 
Authored: Mon Sep 19 23:49:28 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 23:49:28 2016 +0800

--
 .../apache/kylin/job/impl/threadpool/BaseSchedulerTest.java| 1 -
 .../java/org/apache/kylin/provision/BuildCubeWithEngine.java   | 6 --
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java   | 1 -
 .../java/org/apache/kylin/provision/BuildCubeWithStream2.java  | 1 -
 4 files changed, 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c68bba0e/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index 97c9f8d..4e092a1 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -55,7 +55,6 @@ public abstract class BaseSchedulerTest extends 
LocalFileMetadataTestCase {
 
 @After
 public void after() throws Exception {
-DefaultScheduler.destroyInstance();
 cleanupTestMetadata();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c68bba0e/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 31cf0eb..3d60a3c 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -79,7 +79,6 @@ public class BuildCubeWithEngine {
 BuildCubeWithEngine buildCubeWithEngine = new 
BuildCubeWithEngine();
 buildCubeWithEngine.before();
 buildCubeWithEngine.build();
-buildCubeWithEngine.after();
 logger.info("Build is done");
 afterClass();
 logger.info("Going to exit");
@@ -149,11 +148,6 @@ public class BuildCubeWithEngine {
 
 }
 
-
-public void after(){
-DefaultScheduler.destroyInstance();
-}
-
 public static void afterClass() {
 HBaseMetadataTestCase.staticCleanupTestMetadata();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c68bba0e/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 6e5313f..b7c609e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -217,7 +217,6 @@ public class BuildCubeWithStream {
 
 public void after() {
 kafkaServer.stop();
-DefaultScheduler.destroyInstance();
 }
 
 protected void waitForJob(String jobId) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c68bba0e/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
index 2812446..d48a473 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
@@ -235,7 +235,6 @@ public class BuildCubeWithStream2 {
 
 public void after() {
 kafkaServer.stop();
-DefaultScheduler.destroyInstance();
 }
 
 protected void waitForJob(String jobId) {

[08/13] kylin git commit: Revert "KYLIN-1762 discard job when no stream message"

2016-09-19 Thread mahongbin 
Revert "KYLIN-1762 discard job when no stream message"

This reverts commit 1108d9eeccecbccffea0b3f9049151672196c91a.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4de2c0c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4de2c0c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4de2c0c9

Branch: refs/heads/tempmaster
Commit: 4de2c0c97bcf342323b620e58ea9279ec8aa5493
Parents: 87d957f
Author: Hongbin Ma 
Authored: Mon Sep 19 23:50:20 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 23:50:20 2016 +0800

--
 .../job/execution/DefaultChainedExecutable.java |  6 ---
 .../kylin/source/kafka/SeekOffsetStep.java  | 45 +---
 2 files changed, 10 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4de2c0c9/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 39a5f4f..753b389 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -88,7 +88,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 boolean allSucceed = true;
 boolean hasError = false;
 boolean hasRunning = false;
-boolean hasDiscarded = false;
 for (Executable task : jobs) {
 final ExecutableState status = task.getStatus();
 if (status == ExecutableState.ERROR) {
@@ -100,9 +99,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 if (status == ExecutableState.RUNNING) {
 hasRunning = true;
 }
-if (status == ExecutableState.DISCARDED) {
-hasDiscarded = true;
-}
 }
 if (allSucceed) {
 setEndTime(System.currentTimeMillis());
@@ -114,8 +110,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 notifyUserStatusChange(executableContext, 
ExecutableState.ERROR);
 } else if (hasRunning) {
 jobService.updateJobOutput(getId(), ExecutableState.RUNNING, 
null, null);
-} else if (hasDiscarded) {
-jobService.updateJobOutput(getId(), ExecutableState.DISCARDED, 
null, null);
 } else {
 jobService.updateJobOutput(getId(), ExecutableState.READY, 
null, null);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4de2c0c9/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
index 479f1b8..5dca93f 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
@@ -17,10 +17,6 @@
 */
 package org.apache.kylin.source.kafka;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Maps;
-import org.apache.commons.math3.util.MathUtils;
 import org.apache.kylin.source.kafka.util.KafkaClient;
 import org.apache.kylin.source.kafka.util.KafkaOffsetMapping;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -38,7 +34,6 @@ import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -106,39 +101,19 @@ public class SeekOffsetStep extends AbstractExecutable {
 }
 }
 
-long totalStartOffset = 0, totalEndOffset = 0;
-for (Long v : startOffsets.values()) {
-totalStartOffset += v;
-}
-for (Long v : endOffsets.values()) {
-totalEndOffset += v;
-}
+KafkaOffsetMapping.saveOffsetStart(segment, startOffsets);
+KafkaOffsetMapping.saveOffsetEnd(segment, endOffsets);
 
-if (totalEndOffset > totalStartOffset) {
-KafkaOffsetMapping.saveOffsetStart(segment, startOffsets);
-KafkaOffsetMapping.saveOffsetEnd(segment, endOffsets);
-

[05/13] kylin git commit: Revert "KYLIN-1726 allow job discard itself"

2016-09-19 Thread mahongbin 
Revert "KYLIN-1726 allow job discard itself"

This reverts commit aff2df5987e98ee9fd64d4803a8a2dea90013e40.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1093f34d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1093f34d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1093f34d

Branch: refs/heads/tempmaster
Commit: 1093f34d02cda54961dc6354c13d6eef17b9ce13
Parents: f9692fa
Author: Hongbin Ma 
Authored: Mon Sep 19 23:49:57 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 23:49:57 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 14 +--
 .../kylin/job/execution/AbstractExecutable.java |  2 -
 .../job/execution/DefaultChainedExecutable.java |  2 -
 .../kylin/job/execution/ExecuteResult.java  |  4 --
 .../kylin/job/DiscardedTestExecutable.java  | 41 
 .../impl/threadpool/DefaultSchedulerTest.java   | 16 
 6 files changed, 11 insertions(+), 68 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1093f34d/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index d494fcc..11eabce 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -322,26 +322,34 @@ public class CubeManager implements IRealizationProvider {
 Iterator iterator = newSegs.iterator();
 while (iterator.hasNext()) {
 CubeSegment currentSeg = iterator.next();
+boolean found = false;
 for (CubeSegment toRemoveSeg : update.getToRemoveSegs()) {
 if (currentSeg.getUuid().equals(toRemoveSeg.getUuid())) {
-logger.info("Remove segment " + currentSeg.toString());
-
toRemoveResources.add(currentSeg.getStatisticsResourcePath());
 iterator.remove();
-break;
+
toRemoveResources.add(toRemoveSeg.getStatisticsResourcePath());
+found = true;
 }
 }
+if (found == false) {
+logger.error("Segment '" + currentSeg.getName() + "' 
doesn't exist for remove.");
+}
 }
 
 }
 
 if (update.getToUpdateSegs() != null) {
 for (CubeSegment segment : update.getToUpdateSegs()) {
+boolean found = false;
 for (int i = 0; i < newSegs.size(); i++) {
 if (newSegs.get(i).getUuid().equals(segment.getUuid())) {
 newSegs.set(i, segment);
+found = true;
 break;
 }
 }
+if (found == false) {
+logger.error("Segment '" + segment.getName() + "' doesn't 
exist for update.");
+}
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/1093f34d/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index b4ca469..90e4d3c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -74,8 +74,6 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 if (!isDiscarded()) {
 if (result.succeed()) {
 executableManager.updateJobOutput(getId(), 
ExecutableState.SUCCEED, null, result.output());
-} else if (result.discarded()) {
-executableManager.updateJobOutput(getId(), 
ExecutableState.DISCARDED, null, result.output());
 } else {
 executableManager.updateJobOutput(getId(), 
ExecutableState.ERROR, null, result.output());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1093f34d/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 5a57b05..39a5f4f 100644
---

[07/13] kylin git commit: Revert "rename the streaming_table.json"

2016-09-19 Thread mahongbin 
Revert "rename the streaming_table.json"

This reverts commit ffdc5d21d5f46daaf25a0dfed9b3afeef78fc62c.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87d957f1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87d957f1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87d957f1

Branch: refs/heads/tempmaster
Commit: 87d957f12b1c0b67fa94742ecb316b4cb2e7699b
Parents: a70108e
Author: Hongbin Ma 
Authored: Mon Sep 19 23:50:13 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 23:50:13 2016 +0800

--
 .../kafka/DEFAULT.STREAMING_TABLE.json  | 21 
 .../streaming/DEFAULT.STREAMING_TABLE.json  |  6 ++
 2 files changed, 27 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/87d957f1/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
new file mode 100644
index 000..6a64cce
--- /dev/null
+++ b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
@@ -0,0 +1,21 @@
+{
+ 
+  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
+  "name": "DEFAULT.STREAMING_TABLE",
+  "topic": "test_streaming_table_topic_xyz",
+  "timeout": 6,
+  "bufferSize": 65536,
+  "parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
+  "last_modified": 0,
+  "clusters": [
+{
+  "brokers": [
+{
+  "id": 0,
+  "host": "sandbox",
+  "port": 6667
+}
+  ]
+}
+  ]
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/87d957f1/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
new file mode 100644
index 000..85a477b
--- /dev/null
+++ b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
@@ -0,0 +1,6 @@
+{
+  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
+  "name": "DEFAULT.STREAMING_TABLE",
+  "type": "kafka",
+  "last_modified": 0
+}

kylin git commit: Revert "KYLIN-2032 do not need dict for partition col if it is not in dimension"

2016-09-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/master a8a1b6e03 -> e17b11bd8


Revert "KYLIN-2032 do not need dict for partition col if it is not in dimension"

This reverts commit a8a1b6e037767b75a3184f4b5db6f384f622a89e.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e17b11bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e17b11bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e17b11bd

Branch: refs/heads/master
Commit: e17b11bd861a1ce7ecfda37997e0011f7e180fd2
Parents: a8a1b6e
Author: shaofengshi 
Authored: Mon Sep 19 22:58:12 2016 +0800
Committer: shaofengshi 
Committed: Mon Sep 19 22:58:12 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeManager.java  | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e17b11bd/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 2660c9f..57b9510 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -943,6 +943,13 @@ public class CubeManager implements IRealizationProvider {
 }
 }
 
+// add partition column in all case
+if (cubeDesc.getModel().getPartitionDesc() != null) {
+TblColRef partitionCol = 
cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
+if (factDictCols.contains(partitionCol) == false) {
+factDictCols.add(partitionCol);
+}
+}
 return factDictCols;
 }
 }

[2/2] kylin git commit: refactor BuildCubeWithStream

2016-09-19 Thread shaofengshi 
refactor BuildCubeWithStream

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a08dd2e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a08dd2e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a08dd2e0

Branch: refs/heads/master
Commit: a08dd2e03900b321617647d1dbf1c4ee8b4b18c2
Parents: c9dcb07
Author: shaofengshi 
Authored: Mon Sep 19 20:18:35 2016 +0800
Committer: shaofengshi 
Committed: Mon Sep 19 22:16:06 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |   7 +-
 .../kylin/provision/BuildCubeWithStream.java|  10 +-
 .../kylin/provision/BuildCubeWithStream2.java   | 145 +--
 3 files changed, 12 insertions(+), 150 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a08dd2e0/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 9b282e3..9e9df05 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -187,6 +187,7 @@ public class DeployUtil {
 File tmpFile = File.createTempFile(factTableName, "csv");
 FileOutputStream out = new FileOutputStream(tmpFile);
 
+InputStream tempIn = null;
 try {
 if (store.exists(factTablePath)) {
 InputStream oldContent = 
store.getResource(factTablePath).inputStream;
@@ -194,13 +195,15 @@ public class DeployUtil {
 }
 IOUtils.copy(in, out);
 IOUtils.closeQuietly(in);
+IOUtils.closeQuietly(out);
 
 store.deleteResource(factTablePath);
-in = new FileInputStream(tmpFile);
-store.putResource(factTablePath, in, System.currentTimeMillis());
+tempIn = new FileInputStream(tmpFile);
+store.putResource(factTablePath, tempIn, 
System.currentTimeMillis());
 } finally {
 IOUtils.closeQuietly(out);
 IOUtils.closeQuietly(in);
+IOUtils.closeQuietly(tempIn);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a08dd2e0/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 6e5313f..bfe1d0a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -62,10 +62,10 @@ public class BuildCubeWithStream {
 
 private static final Logger logger = 
LoggerFactory.getLogger(org.apache.kylin.provision.BuildCubeWithStream.class);
 
-private CubeManager cubeManager;
+protected CubeManager cubeManager;
 private DefaultScheduler scheduler;
 protected ExecutableManager jobService;
-private static final String cubeName = "test_streaming_table_cube";
+static final String cubeName = "test_streaming_table_cube";
 
 private KafkaConfig kafkaConfig;
 private MockKafka kafkaServer;
@@ -114,13 +114,13 @@ public class BuildCubeWithStream {
 Assert.assertEquals(topicName, topicMetadata.topic());
 }
 
-private void generateStreamData(long startTime, long endTime, int 
numberOfRecords) throws IOException {
+protected void generateStreamData(long startTime, long endTime, int 
numberOfRecords) throws IOException {
 Kafka10DataLoader dataLoader = new Kafka10DataLoader(kafkaConfig);
 DeployUtil.prepareTestDataForStreamingCube(startTime, endTime, 
numberOfRecords, cubeName, dataLoader);
 logger.info("Test data inserted into Kafka");
 }
 
-private void clearSegment(String cubeName) throws Exception {
+protected void clearSegment(String cubeName) throws Exception {
 CubeInstance cube = cubeManager.getCube(cubeName);
 // remove all existing segments
 CubeUpdate cubeBuilder = new CubeUpdate(cube);
@@ -187,7 +187,7 @@ public class BuildCubeWithStream {
 return job.getId();
 }
 
-private ExecutableState buildSegment(String cubeName, long startOffset, 
long endOffset) throws Exception {
+protected ExecutableState buildSegment(String cubeName, long startOffset, 
long endOffset) throws Exception {
 CubeSegment segment = 
cubeManager.appendSegment(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset);
 DefaultChainedExecutable job =

[1/2] kylin git commit: KYLIN-2032 do not need dict for partition col if it is not in dimension

2016-09-19 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/master c9dcb07c7 -> a8a1b6e03


KYLIN-2032 do not need dict for partition col if it is not in dimension

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8a1b6e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8a1b6e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8a1b6e0

Branch: refs/heads/master
Commit: a8a1b6e037767b75a3184f4b5db6f384f622a89e
Parents: a08dd2e
Author: shaofengshi 
Authored: Mon Sep 19 22:15:50 2016 +0800
Committer: shaofengshi 
Committed: Mon Sep 19 22:16:06 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/CubeManager.java  | 7 ---
 1 file changed, 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a8a1b6e0/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 57b9510..2660c9f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -943,13 +943,6 @@ public class CubeManager implements IRealizationProvider {
 }
 }
 
-// add partition column in all case
-if (cubeDesc.getModel().getPartitionDesc() != null) {
-TblColRef partitionCol = 
cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
-if (factDictCols.contains(partitionCol) == false) {
-factDictCols.add(partitionCol);
-}
-}
 return factDictCols;
 }
 }

kylin git commit: KYLIN-1726 debugging messages

2016-09-19 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 9ebc741d8 -> c9dcb07c7


KYLIN-1726 debugging messages


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9dcb07c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9dcb07c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9dcb07c

Branch: refs/heads/master
Commit: c9dcb07c72d0fef99ce8f630936b819c2db73408
Parents: 9ebc741
Author: Hongbin Ma 
Authored: Mon Sep 19 22:11:59 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Sep 19 22:13:13 2016 +0800

--
 .../org/apache/kylin/source/kafka/TimedJsonStreamParser.java   | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c9dcb07c/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index 7db8285..ab8b161 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -18,14 +18,15 @@
 package org.apache.kylin.source.kafka;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+
 import org.apache.commons.lang3.StringUtils;
-import java.nio.ByteBuffer;
 import org.apache.kylin.common.util.StreamingMessage;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
@@ -103,7 +104,8 @@ public final class TimedJsonStreamParser extends 
StreamingParser {
 }
 }
 
-return new StreamingMessage(result, 0, t, Collections.emptyMap());
+logger.info("Streaming Message: " + result.toString());
+return new StreamingMessage(result, 0, t, Collections. emptyMap());
 } catch (IOException e) {
 logger.error("error", e);
 throw new RuntimeException(e);

kylin git commit: state that windows and mac have known issues

2016-09-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/document b636a616f -> 5f09278ee


state that windows and mac have known issues


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f09278e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f09278e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f09278e

Branch: refs/heads/document
Commit: 5f09278ee7cd9405d6b2b735fdc36fa84481faf6
Parents: b636a61
Author: Li Yang 
Authored: Mon Sep 19 21:48:48 2016 +0800
Committer: Li Yang 
Committed: Mon Sep 19 21:48:48 2016 +0800

--
 website/_docs15/install/hadoop_evn.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5f09278e/website/_docs15/install/hadoop_evn.md
--
diff --git a/website/_docs15/install/hadoop_evn.md 
b/website/_docs15/install/hadoop_evn.md
index 10c98dc..45bd3b5 100644
--- a/website/_docs15/install/hadoop_evn.md
+++ b/website/_docs15/install/hadoop_evn.md
@@ -14,9 +14,9 @@ Kylin need run in a Hadoop node, to get better stability, we 
suggest you to depl
 * HBase: 0.98 - 0.99, 1.x
 * JDK: 1.7+
 
-_Tested with Hortonworks HDP 2.2 and Cloudera Quickstart VM 5.1_
+_Tested with Hortonworks HDP 2.2 and Cloudera Quickstart VM 5.1. Windows and 
MacOS have known issues._
 
-To make things easier we strongly recommend you try Kylin with an all-in-one 
sandbox VM, like [HDP 
sandbox](http://hortonworks.com/products/hortonworks-sandbox/). In the 
following tutorial we'll go with **Hortonworks Sandbox 2.1** and **Cloudera 
QuickStart VM 5.1**. 
+To make things easier we strongly recommend you try Kylin with an all-in-one 
sandbox VM, like [HDP 
sandbox](http://hortonworks.com/products/hortonworks-sandbox/), and give it 10 
GB memory. In the following tutorial we'll go with **Hortonworks Sandbox 2.1** 
and **Cloudera QuickStart VM 5.1**. 
 
 To avoid permission issue in the sandbox, you can use its `root` account. The 
password for **Hortonworks Sandbox 2.1** is `hadoop` , for **Cloudera 
QuickStart VM 5.1** is `cloudera`.

kylin git commit: KYLIN-2028 output-delimiter option to sed command

2016-09-19 Thread liyang 
Repository: kylin
Updated Branches:
  refs/heads/master 805b938d5 -> 9ebc741d8


KYLIN-2028 output-delimiter option to sed command

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ebc741d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ebc741d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ebc741d

Branch: refs/heads/master
Commit: 9ebc741d871ce5cb679d440b8e6adf9ac9cb6692
Parents: 805b938
Author: Roger Shi 
Authored: Mon Sep 19 14:18:33 2016 +0800
Committer: Li Yang 
Committed: Mon Sep 19 15:54:36 2016 +0800

--
 build/bin/find-hbase-dependency.sh | 2 +-
 build/bin/find-hive-dependency.sh  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ebc741d/build/bin/find-hbase-dependency.sh
--
diff --git a/build/bin/find-hbase-dependency.sh 
b/build/bin/find-hbase-dependency.sh
index c29de43..ea9a2e2 100644
--- a/build/bin/find-hbase-dependency.sh
+++ b/build/bin/find-hbase-dependency.sh
@@ -25,7 +25,7 @@ if [ -n is_aws ] && [ -d "/usr/lib/oozie/lib" ]; then
 export HBASE_ENV_INIT="true"
 fi
 
-arr=(`echo $hbase_classpath | cut -d ":"  --output-delimiter=" " -f 1-`)
+arr=(`echo $hbase_classpath | cut -d ":" -f 1- | sed 's/:/ /g'`)
 hbase_common_path=
 for data in ${arr[@]}
 do

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ebc741d/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh 
b/build/bin/find-hive-dependency.sh
index 71c2fe6..d17b47e 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -38,7 +38,7 @@ else
 fi
 
 hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk -F '=' '{print 
$2}'`
-arr=(`echo $hive_classpath | cut -d ":"  --output-delimiter=" " -f 1-`)
+arr=(`echo $hive_classpath | cut -d ":" -f 1- | sed 's/:/ /g'`)
 hive_conf_path=
 hive_exec_path=