[2/3] kylin git commit: KYLIN-1297 avoid unnecessary converter creation

[mahongbin]

KYLIN-1297 avoid unnecessary converter creation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b4ea1b39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b4ea1b39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b4ea1b39

Branch: refs/heads/2.x-staging
Commit: b4ea1b3915c70daf44a871a39d65282e9089b77c
Parents: 32cb630
Author: honma 
Authored: Thu Jan 14 18:34:35 2016 +0800
Committer: honma 
Committed: Thu Jan 14 18:34:35 2016 +0800

--
 .../storage/hbase/cube/v2/SequentialCubeTupleIterator.java | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b4ea1b39/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
index 7e70a4d..dcc3aba 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
@@ -56,6 +56,7 @@ public class SequentialCubeTupleIterator implements 
ITupleIterator {
 
 @Override
 public boolean hasNext() {
+logger.info("hasNext called");
 if (next != null)
 return true;
 
@@ -75,7 +76,10 @@ public class SequentialCubeTupleIterator implements 
ITupleIterator {
 if (scannerIterator.hasNext()) {
 curScanner = scannerIterator.next();
 curRecordIterator = curScanner.iterator();
-curTupleConverter = new CubeTupleConverter(curScanner.cubeSeg, 
cuboid, selectedDimensions, selectedMetrics, tupleInfo);
+if (curRecordIterator.hasNext()) {
+//if the segment does not has any tuples, don't bother to 
create a converter
+curTupleConverter = new 
CubeTupleConverter(curScanner.cubeSeg, cuboid, selectedDimensions, 
selectedMetrics, tupleInfo);
+}
 } else {
 return false;
 }

[1/3] kylin git commit: don't do storage cleaning at buildcubewithengine

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 08f42c0d5 -> d2bc9d140


don't do storage cleaning at buildcubewithengine


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32cb630d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32cb630d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32cb630d

Branch: refs/heads/2.x-staging
Commit: 32cb630d746e5f0948591fec0d5f52bbc8ef6dfb
Parents: 08f42c0
Author: honma 
Authored: Thu Jan 14 17:39:23 2016 +0800
Committer: honma 
Committed: Thu Jan 14 17:41:10 2016 +0800

--
 .../src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/32cb630d/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 46bf1bb..331e21d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -107,7 +107,6 @@ public class BuildCubeWithEngineTest {
 public void before() throws Exception {
 
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
 
-cleanupOldStorage();
 DeployUtil.initCliWorkDir();
 DeployUtil.deployMetadata();
 DeployUtil.overrideJobJarLocations();

kylin git commit: KYLIN-1303 Support tinyint in DataTypeSerializer

[lidong]

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 6ab821c94 -> a09df13d3


KYLIN-1303 Support tinyint in DataTypeSerializer


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a09df13d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a09df13d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a09df13d

Branch: refs/heads/2.0-rc
Commit: a09df13d354afc5dcf63b99b4967227d5afb8074
Parents: 6ab821c
Author: lidongsjtu 
Authored: Fri Jan 15 09:54:14 2016 +0800
Committer: lidongsjtu 
Committed: Fri Jan 15 09:54:14 2016 +0800

--
 .../kylin/metadata/measure/serializer/DataTypeSerializer.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a09df13d/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
index 364830a..d42233d 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
@@ -48,6 +48,7 @@ abstract public class DataTypeSerializer implements 
BytesSerializer {
 impl.put("long", LongSerializer.class);
 impl.put("integer", LongSerializer.class);
 impl.put("int", LongSerializer.class);
+impl.put("tinyint", LongSerializer.class);
 impl.put("smallint", LongSerializer.class);
 impl.put("boolean", BooleanSerializer.class);
 impl.put("date", DateTimeSerializer.class);

[8/8] kylin git commit: KYLIN-1311 fix unit tests after rebase

[shaofengshi]

KYLIN-1311 fix unit tests after rebase

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c66114e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c66114e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c66114e

Branch: refs/heads/helix-201601
Commit: 2c66114ee237ea2589d8f7adbf6ec1b7c0cb3dbb
Parents: b2eab0c
Author: shaofengshi 
Authored: Fri Jan 15 14:44:27 2016 +0800
Committer: shaofengshi 
Committed: Fri Jan 15 14:44:27 2016 +0800

--
 .../engine/spark/BuildCubeWithSparkTest.java|   2 +-
 .../kylin/job/BuildCubeWithEngineTest.java  |   2 +-
 .../apache/kylin/job/BuildIIWithEngineTest.java |   2 +-
 build/conf/kylin.properties |  15 +-
 .../apache/kylin/common/KylinConfigBase.java|   6 +-
 .../job/impl/threadpool/DefaultScheduler.java   |  27 +-
 .../job/impl/threadpool/BaseSchedulerTest.java  |   2 +-
 .../test_case_data/sandbox/kylin.properties |  10 +-
 pom.xml |  11 +-
 server/pom.xml  |  32 +++
 .../java/org/apache/kylin/rest/DebugTomcat.java |   4 +-
 .../kylin/rest/controller/JobController.java|  50 ++--
 .../kylin/rest/helix/HelixClusterAdmin.java |  25 +-
 .../apache/kylin/rest/service/CubeService.java  |   7 +-
 .../rest/controller/JobControllerTest.java  | 245 ++-
 .../kylin/rest/helix/HelixClusterAdminTest.java |   4 +-
 .../kylin/rest/service/CacheServiceTest.java|  18 --
 .../kylin/storage/hbase/HBaseConnection.java|  17 ++
 .../storage/hbase/util/ZookeeperJobLock.java|  25 +-
 19 files changed, 288 insertions(+), 216 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c66114e/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
 
b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index f5b9741..523033e 100644
--- 
a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -97,7 +97,7 @@ public class BuildCubeWithSparkTest {
 for (String jobId : jobService.getAllJobIds()) {
 jobService.deleteJob(jobId);
 }
-scheduler = DefaultScheduler.getInstance();
+scheduler = DefaultScheduler.createInstance();
 scheduler.init(new JobEngineConfig(kylinConfig), new MockJobLock());
 if (!scheduler.hasStarted()) {
 throw new RuntimeException("scheduler has not been started");

http://git-wip-us.apache.org/repos/asf/kylin/blob/2c66114e/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 46bf1bb..3c44a43 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -114,7 +114,7 @@ public class BuildCubeWithEngineTest {
 
 final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 jobService = ExecutableManager.getInstance(kylinConfig);
-scheduler = DefaultScheduler.getInstance();
+scheduler = DefaultScheduler.createInstance();
 scheduler.init(new JobEngineConfig(kylinConfig), new 
ZookeeperJobLock());
 if (!scheduler.hasStarted()) {
 throw new RuntimeException("scheduler has not been started");

http://git-wip-us.apache.org/repos/asf/kylin/blob/2c66114e/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java 
b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 0158fad..ef8cf42 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -109,7 +109,7 @@ public class BuildIIWithEngineTest {
 
 final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 jobService = ExecutableManager.getInstance(kylinConfig);
-scheduler = DefaultScheduler.getInstance();
+scheduler = DefaultScheduler.createInstance();
 scheduler.init(new JobEngineConfig(kylinConfig), new 
ZookeeperJobLock());
 if 

[6/8] kylin git commit: KYLIN-1188 use helix 0.7.1 to manage the job engine assignment

[shaofengshi]

KYLIN-1188 use helix 0.7.1 to manage the job engine assignment


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a61d4779
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a61d4779
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a61d4779

Branch: refs/heads/helix-201601
Commit: a61d4779695999db5d62c5d12c6a2504ff77e9cb
Parents: 2304b94
Author: shaofengshi 
Authored: Tue Jan 12 15:07:25 2016 +0800
Committer: shaofengshi 
Committed: Thu Jan 14 15:27:46 2016 +0800

--
 build/conf/kylin.properties |  16 +-
 .../apache/kylin/common/KylinConfigBase.java|  28 +++
 .../test_case_data/sandbox/kylin.properties |   2 +
 pom.xml |   1 +
 .../kylin/rest/controller/JobController.java|  33 +--
 .../kylin/rest/helix/HelixClusterAdmin.java | 245 +++
 .../helix/LeaderStandbyStateModelFactory.java   |  70 ++
 .../apache/kylin/rest/service/CubeService.java  |   6 +-
 .../kylin/rest/helix/HelixClusterAdminTest.java | 140 +++
 9 files changed, 516 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a61d4779/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 44a282e..8456ecb 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -1,12 +1,24 @@
-## Config for Kylin Engine ##
+## Cluster related properties ##
+# Required, comma separated list of zk servers; 
+kylin.zookeeper.address=
 
+# rest address of this instance, ;
+# optional, default be :7070
+kylin.rest.address=
+
+# whether run a cluster controller in this node
+kylin.cluster.controller=true
 
 # optional information for the owner of kylin platform, it can be your team's 
email
 # currently it will be attached to each kylin's htable attribute
 kylin.owner=who...@kylin.apache.org
 
 # List of web servers in use, this enables one web server instance to sync up 
with other servers.
-kylin.rest.servers=localhost:7070
+# Deprecated, cluster will self-discover and update this.
+# kylin.rest.servers=localhost:7070
+
+# Server mode: all, job, query
+kylin.server.mode=all
 
 # The metadata store in hbase
 kylin.metadata.url=kylin_metadata@hbase

http://git-wip-us.apache.org/repos/asf/kylin/blob/a61d4779/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index bfad306..414a750 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -519,6 +519,34 @@ public class KylinConfigBase implements Serializable {
 return getOptional("mail.sender", "");
 }
 
+public String getZookeeperAddress() {
+return this.getOptional("kylin.zookeeper.address");
+}
+
+public void setZookeeperAddress(String zkAddress) {
+setProperty("kylin.zookeeper.address", zkAddress);
+}
+
+public String getClusterName() {
+return this.getOptional("kylin.cluster.name", getMetadataUrlPrefix());
+}
+
+public void setClusterName(String clusterName) {
+setProperty("kylin.cluster.name", clusterName);
+}
+
+public boolean isClusterController() {
+return Boolean.parseBoolean(getOptional("kylin.cluster.controller", 
"true"));
+}
+
+public String getRestAddress() {
+return this.getOptional("kylin.rest.address");
+}
+
+public void setRestAddress(String restAddress) {
+setProperty("kylin.rest.address", restAddress);
+}
+
 public String toString() {
 return getMetadataUrl();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a61d4779/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 18ff1cc..5ce636b 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -11,6 +11,8 @@ kylin.rest.servers=localhost:7070
 #set display timezone on UI,format like[GMT+N or GMT-N]
 kylin.rest.timezone=GMT-8
 
+kylin.server.mode=all
+>>> KYLIN-1188 use helix 0.7.1 to manage the job engine assignment
 # The metadata store in hbase
 kylin.metadata.url=kylin_metadata@hbase
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a61d4779/pom.xml

[3/8] kylin git commit: rebase 2.x-staging

[shaofengshi]

rebase 2.x-staging


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c891cd42
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c891cd42
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c891cd42

Branch: refs/heads/helix-201601
Commit: c891cd428ecc2473fb7bee246fb8d23372ccafec
Parents: 2e8a692
Author: shaofengshi 
Authored: Wed Dec 30 14:22:35 2015 +0800
Committer: shaofengshi 
Committed: Thu Jan 14 15:22:03 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  4 +++
 .../kylin/rest/controller/JobController.java| 26 +++-
 .../apache/kylin/rest/service/CubeService.java  |  8 +++---
 3 files changed, 18 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c891cd42/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 81f5827..ea77e47 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -264,5 +264,9 @@ public class KylinConfig extends KylinConfigBase {
 out.println(key + "=" + val);
 }
 }
+
+public String getClusterName() {
+return this.getOptional("kylin.cluster.name", getMetadataUrlPrefix());
+}
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c891cd42/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 4d0824a..9dfb594 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -26,18 +26,15 @@ import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
 
-import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import joptsimple.internal.Strings;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.restclient.Broadcaster;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.constant.JobStatusEnum;
 import org.apache.kylin.job.constant.JobTimeFilterEnum;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.lock.JobLock;
-import org.apache.kylin.rest.constant.Constant;
 import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.helix.HelixJobEngineAdmin;
 import org.apache.kylin.rest.request.JobListRequest;
 import org.apache.kylin.rest.service.JobService;
 import org.slf4j.Logger;
@@ -50,6 +47,9 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.ResponseBody;
 
+import java.io.IOException;
+import java.util.*;
+
 /**
  * @author ysong1
  * @author Jack
@@ -63,9 +63,6 @@ public class JobController extends BasicController implements 
InitializingBean {
 @Autowired
 private JobService jobService;
 
-@Autowired
-private JobLock jobLock;
-
 /*
  * (non-Javadoc)
  * 
@@ -79,15 +76,9 @@ public class JobController extends BasicController 
implements InitializingBean {
 TimeZone tzone = TimeZone.getTimeZone(timeZone);
 TimeZone.setDefault(tzone);
 
-if (System.getProperty("kylin.rest.address") == null) {
-throw new RuntimeException("There is no -Dkylin.rest.address set; 
Please check bin/kylin.sh");
-}
-
-final String restAddress = System.getProperty("kylin.rest.address");
-final String hostname = 
Preconditions.checkNotNull(restAddress.substring(0, 
restAddress.lastIndexOf(":")));
-final String port = 
Preconditions.checkNotNull(restAddress.substring(restAddress.lastIndexOf(":") + 
1));
-final String instanceName = hostname + "_" + port;
+final String instanceName = 
HelixJobEngineAdmin.getCurrentInstanceName();
 final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+
 }
 
 /**
@@ -199,7 +190,7 @@ public class JobController extends BasicController 
implements InitializingBean {
 this.jobService = jobService;
 }
 
-private void updateKylinConfig(List instances) {
+private void updateKylinCluster(List instances) {
 List instanceRestAddresses = 

[5/8] kylin git commit: KYLIN-1311 Stream cubing auto assignment and load balance

[shaofengshi]

KYLIN-1311 Stream cubing auto assignment and load balance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db133690
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db133690
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db133690

Branch: refs/heads/helix-201601
Commit: db133690487e5b3fe7d7c8e8336c42a2647de158
Parents: a61d477
Author: shaofengshi 
Authored: Wed Jan 13 12:00:48 2016 +0800
Committer: shaofengshi 
Committed: Thu Jan 14 15:27:46 2016 +0800

--
 .../apache/kylin/rest/constant/Constant.java|  1 +
 .../kylin/rest/helix/HelixClusterAdmin.java | 22 +++--
 .../helix/LeaderStandbyStateModelFactory.java   | 50 +++-
 3 files changed, 68 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db133690/server/src/main/java/org/apache/kylin/rest/constant/Constant.java
--
diff --git a/server/src/main/java/org/apache/kylin/rest/constant/Constant.java 
b/server/src/main/java/org/apache/kylin/rest/constant/Constant.java
index f068e5f..58b74f0 100644
--- a/server/src/main/java/org/apache/kylin/rest/constant/Constant.java
+++ b/server/src/main/java/org/apache/kylin/rest/constant/Constant.java
@@ -41,6 +41,7 @@ public class Constant {
 
 public final static String SERVER_MODE_QUERY = "query";
 public final static String SERVER_MODE_JOB = "job";
+public final static String SERVER_MODE_STREAM = "stream";
 public final static String SERVER_MODE_ALL = "all";
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/db133690/server/src/main/java/org/apache/kylin/rest/helix/HelixClusterAdmin.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/helix/HelixClusterAdmin.java 
b/server/src/main/java/org/apache/kylin/rest/helix/HelixClusterAdmin.java
index 9983aae..6300383 100644
--- a/server/src/main/java/org/apache/kylin/rest/helix/HelixClusterAdmin.java
+++ b/server/src/main/java/org/apache/kylin/rest/helix/HelixClusterAdmin.java
@@ -45,10 +45,12 @@ import java.util.concurrent.ConcurrentMap;
 public class HelixClusterAdmin {
 
 public static final String RESOURCE_NAME_JOB_ENGINE = "Resource_JobEngine";
+public static final String RESOURCE_STREAME_CUBE_PREFIX = 
"Resource_Streame_";
 
 public static final String MODEL_LEADER_STANDBY = "LeaderStandby";
 public static final String MODEL_ONLINE_OFFLINE = "OnlineOffline";
 public static final String TAG_JOB_ENGINE = "Tag_JobEngine";
+public static final String TAG_STREAM_BUILDER = "Tag_StreamBuilder";
 
 private static ConcurrentMap instanceMaps 
= Maps.newConcurrentMap();
 private HelixManager participantManager;
@@ -74,11 +76,15 @@ public class HelixClusterAdmin {
 
 // use the tag to mark node's role.
 final List instanceTags = Lists.newArrayList();
-final boolean runJobEngine = 
Constant.SERVER_MODE_ALL.equalsIgnoreCase(kylinConfig.getServerMode()) || 
Constant.SERVER_MODE_JOB.equalsIgnoreCase(kylinConfig.getServerMode());
-if (runJobEngine) {
+if 
(Constant.SERVER_MODE_ALL.equalsIgnoreCase(kylinConfig.getServerMode())) {
 instanceTags.add(HelixClusterAdmin.TAG_JOB_ENGINE);
+instanceTags.add(HelixClusterAdmin.TAG_STREAM_BUILDER);
+} else if 
(Constant.SERVER_MODE_JOB.equalsIgnoreCase(kylinConfig.getServerMode())) {
+instanceTags.add(HelixClusterAdmin.TAG_JOB_ENGINE);
+} else if 
(Constant.SERVER_MODE_STREAM.equalsIgnoreCase(kylinConfig.getServerMode())) {
+instanceTags.add(HelixClusterAdmin.TAG_STREAM_BUILDER);
 }
-
+
 addInstance(instanceName, instanceTags);
 startInstance(instanceName);
 
@@ -108,6 +114,16 @@ public class HelixClusterAdmin {
 }
 
 }
+
+public void addStreamCubeSlice(String cubeName, long start, long end) {
+String resourceName = RESOURCE_STREAME_CUBE_PREFIX + cubeName + "_" + 
start + "_" + end;
+if (!admin.getResourcesInCluster(clusterName).contains(resourceName)) {
+admin.addResource(clusterName, resourceName, 1, 
MODEL_LEADER_STANDBY, IdealState.RebalanceMode.SEMI_AUTO.name());
+}
+
+admin.rebalance(clusterName, resourceName, 2, "", TAG_STREAM_BUILDER);
+
+}
 
 /**
  * Start the instance and register the state model factory

http://git-wip-us.apache.org/repos/asf/kylin/blob/db133690/server/src/main/java/org/apache/kylin/rest/helix/LeaderStandbyStateModelFactory.java
--
diff --git 

[7/8] kylin git commit: KYLIN-1311 Stream cubing auto assignment and load balance

[shaofengshi]

KYLIN-1311 Stream cubing auto assignment and load balance


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2eab0c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2eab0c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2eab0c7

Branch: refs/heads/helix-201601
Commit: b2eab0c7c3f34b4ec5230ace13b05a4021ca32a8
Parents: db13369
Author: shaofengshi 
Authored: Thu Jan 14 14:59:54 2016 +0800
Committer: shaofengshi 
Committed: Thu Jan 14 15:27:46 2016 +0800

--
 .../kylin/engine/streaming/BootstrapConfig.java |  8 --
 .../engine/streaming/cli/StreamingCLI.java  |  3 -
 .../kylin/rest/controller/CubeController.java   |  6 ++
 .../rest/controller/StreamingController.java| 50 +
 .../kylin/rest/helix/HelixClusterAdmin.java | 13 +++-
 .../helix/LeaderStandbyStateModelFactory.java   | 43 +++
 .../rest/request/StreamingBuildRequest.java | 77 
 .../kylin/rest/request/StreamingRequest.java|  4 +-
 .../kylin/rest/service/StreamingService.java| 27 +++
 9 files changed, 202 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b2eab0c7/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
index a3e2db5..2b83b84 100644
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
+++ 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/BootstrapConfig.java
@@ -36,14 +36,6 @@ public class BootstrapConfig {
 this.streaming = streaming;
 }
 
-public int getPartitionId() {
-return partitionId;
-}
-
-public void setPartitionId(int partitionId) {
-this.partitionId = partitionId;
-}
-
 public boolean isFillGap() {
 return fillGap;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b2eab0c7/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
--
diff --git 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
index a73a6ac..96ad1ad 100644
--- 
a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
+++ 
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cli/StreamingCLI.java
@@ -72,9 +72,6 @@ public class StreamingCLI {
 case "-streaming":
 bootstrapConfig.setStreaming(args[++i]);
 break;
-case "-partition":
-
bootstrapConfig.setPartitionId(Integer.parseInt(args[++i]));
-break;
 case "-fillGap":
 
bootstrapConfig.setFillGap(Boolean.parseBoolean(args[++i]));
 break;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b2eab0c7/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f9ac14e..0231432 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,14 +27,19 @@ import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.CubeUpdate;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.engine.streaming.BootstrapConfig;
 import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.engine.streaming.StreamingManager;
+import org.apache.kylin.engine.streaming.monitor.StreamingMonitor;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.JoinedFlatTable;
 import org.apache.kylin.job.exception.JobException;
@@ -780,4 +785,5 @@ public class CubeController extends BasicController {
 public void 

[4/8] kylin git commit: remove getClusterName() from KylinConfig.java

[shaofengshi]

remove getClusterName() from KylinConfig.java


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2304b941
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2304b941
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2304b941

Branch: refs/heads/helix-201601
Commit: 2304b941aa763131eae244ae7b5d3b54a3739f57
Parents: 6e92dbd
Author: shaofengshi 
Authored: Wed Dec 30 16:09:32 2015 +0800
Committer: shaofengshi 
Committed: Thu Jan 14 15:22:03 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfig.java   | 4 
 1 file changed, 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2304b941/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index ea77e47..81f5827 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -264,9 +264,5 @@ public class KylinConfig extends KylinConfigBase {
 out.println(key + "=" + val);
 }
 }
-
-public String getClusterName() {
-return this.getOptional("kylin.cluster.name", getMetadataUrlPrefix());
-}
 
 }