KYLIN-1311 fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1ccbb38 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1ccbb38 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1ccbb38 Branch: refs/heads/helix-rebase Commit: a1ccbb3825b5c24f6e43bced4ee5e27dff19d98b Parents: f778ab4 Author: shaofengshi <shaofeng...@apache.org> Authored: Wed Feb 10 21:23:35 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Wed Mar 2 17:27:46 2016 +0800 ---------------------------------------------------------------------- .../rest/controller/ClusterController.java | 6 +++- .../rest/controller/StreamingController.java | 6 ++-- .../helix/StreamCubeBuildTransitionHandler.java | 12 +++++-- .../apache/kylin/rest/service/CubeService.java | 11 ++++-- .../kylin/rest/service/StreamingService.java | 13 ++++--- .../rest/controller/UserControllerTest.java | 12 ++----- .../kylin/rest/helix/HelixClusterAdminTest.java | 6 +++- .../kylin/rest/service/CacheServiceTest.java | 2 -- .../kylin/rest/service/ServiceTestBase.java | 36 ++++++++------------ .../rest/service/TestBaseWithZookeeper.java | 9 ++--- 10 files changed, 56 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java b/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java index 86a0398..51db0a7 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java @@ -25,6 +25,7 @@ import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.engine.streaming.StreamingManager; import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.job.impl.threadpool.DefaultScheduler; +import org.apache.kylin.job.lock.JobLock; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.helix.HelixClusterAdmin; import org.apache.kylin.rest.request.StreamingBuildRequest; @@ -32,6 +33,7 @@ import org.apache.kylin.storage.hbase.util.ZookeeperJobLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -48,6 +50,8 @@ import java.util.Collection; public class ClusterController extends BasicController implements InitializingBean { private static final Logger logger = LoggerFactory.getLogger(ClusterController.class); + @Autowired + private JobLock jobLock; /* * (non-Javadoc) * @@ -78,7 +82,7 @@ public class ClusterController extends BasicController implements InitializingBe public void run() { try { DefaultScheduler scheduler = DefaultScheduler.createInstance(); - scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock()); + scheduler.init(new JobEngineConfig(kylinConfig), jobLock); if (!scheduler.hasStarted()) { logger.error("scheduler has not been started"); System.exit(1); http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java index e33a1c9..74b0dae 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java @@ -253,8 +253,10 @@ public class StreamingController extends BasicController { streamingBuildRequest.setStreaming(streamingConfig.getName()); try { streamingService.buildStream(cube, streamingBuildRequest); - } catch (IOException e) { - e.printStackTrace(); + } catch (IOException e) { + logger.error("", e); + streamingBuildRequest.setSuccessful(false); + streamingBuildRequest.setMessage("Failed to submit job for " + streamingBuildRequest.getStreaming() + ", error is: " + e.getMessage()); return streamingBuildRequest; } streamingBuildRequest.setMessage("Build request is submitted successfully."); http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java b/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java index 705d8a7..4652d0d 100644 --- a/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java +++ b/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java @@ -2,6 +2,7 @@ package org.apache.kylin.rest.helix; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import org.apache.commons.io.IOUtils; import org.apache.helix.NotificationContext; import org.apache.helix.api.TransitionHandler; import org.apache.helix.model.Message; @@ -120,17 +121,22 @@ public class StreamCubeBuildTransitionHandler extends TransitionHandler { private void runCMD(String cmd) { logger.info("Executing: " + cmd); + BufferedReader input = null; + Process p = null; try { String line; - Process p = Runtime.getRuntime().exec(cmd); - BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream())); + p = Runtime.getRuntime().exec(cmd); + input = new BufferedReader(new InputStreamReader(p.getInputStream())); while ((line = input.readLine()) != null) { logger.info(line); } - input.close(); + + logger.info("Successfully start: " + cmd); } catch (IOException err) { logger.error("Error happens when running '" + cmd + "'", err); throw new RuntimeException(err); + } finally { + IOUtils.closeQuietly(input); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index e7411a9..b2a278a 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -590,10 +590,15 @@ public class CubeService extends BasicService { public void updateOnNewSegmentReady(String cubeName) { logger.debug("on updateOnNewSegmentReady: " + cubeName); final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - boolean isLeaderRole = true; + boolean isLeaderRole = false; if (kylinConfig.isClusterEnabled()) { - HelixClusterAdmin jobEngineAdmin = HelixClusterAdmin.getInstance(kylinConfig); - isLeaderRole = jobEngineAdmin.isLeaderRole(HelixClusterAdmin.RESOURCE_NAME_JOB_ENGINE); + HelixClusterAdmin clusterAdmin = HelixClusterAdmin.getInstance(kylinConfig); + isLeaderRole = clusterAdmin.isLeaderRole(HelixClusterAdmin.RESOURCE_NAME_JOB_ENGINE); + } else { + String serverMode = kylinConfig.getServerMode(); + if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase())) { + isLeaderRole = true; + } } logger.debug("server is leader role ? " + isLeaderRole); if (isLeaderRole == true) { http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java b/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java index 6e732d9..28b9472 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java @@ -101,14 +101,13 @@ public class StreamingService extends BasicService { @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") public void buildStream(CubeInstance cube, StreamingBuildRequest streamingBuildRequest) throws IOException { - HelixClusterAdmin clusterAdmin = HelixClusterAdmin.getInstance(KylinConfig.getInstanceFromEnv()); - try { - clusterAdmin.addStreamingJob(streamingBuildRequest); - } catch (IOException e) { - logger.error("", e); - streamingBuildRequest.setSuccessful(false); - streamingBuildRequest.setMessage("Failed to submit job for " + streamingBuildRequest.getStreaming()); + final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + if (kylinConfig.isClusterEnabled() == false) { + throw new IllegalStateException("Set kylin.cluster.enabled to true to enable streaming feature."); } + + HelixClusterAdmin clusterAdmin = HelixClusterAdmin.getInstance(kylinConfig); + clusterAdmin.addStreamingJob(streamingBuildRequest); } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java index fe0e67a..2c4a1cb 100644 --- a/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java +++ b/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java @@ -18,22 +18,14 @@ package org.apache.kylin.rest.controller; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - import org.apache.kylin.rest.service.ServiceTestBase; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import org.springframework.security.authentication.TestingAuthenticationToken; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.GrantedAuthority; -import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.security.core.userdetails.User; import org.springframework.security.core.userdetails.UserDetails; +import java.io.IOException; + /** * @author xduo */ http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java b/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java index 1c8b779..24a8bb3 100644 --- a/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java +++ b/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java @@ -22,6 +22,7 @@ import org.I0Itec.zkclient.ZkClient; import org.I0Itec.zkclient.ZkServer; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileUtil; +import org.apache.helix.HelixAdmin; import org.apache.helix.manager.zk.ZKHelixAdmin; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.LocalFileMetadataTestCase; @@ -44,6 +45,7 @@ public class HelixClusterAdminTest extends TestBaseWithZookeeper { HelixClusterAdmin clusterAdmin1; HelixClusterAdmin clusterAdmin2; + HelixAdmin zkHelixAdmin; KylinConfig kylinConfig; private static final String CLUSTER_NAME = "test_cluster"; @@ -54,7 +56,7 @@ public class HelixClusterAdminTest extends TestBaseWithZookeeper { kylinConfig.setRestAddress("localhost:7070"); kylinConfig.setClusterName(CLUSTER_NAME); - final ZKHelixAdmin zkHelixAdmin = new ZKHelixAdmin(zkAddress); + zkHelixAdmin = new ZKHelixAdmin(zkAddress); zkHelixAdmin.dropCluster(kylinConfig.getClusterName()); } @@ -117,6 +119,8 @@ public class HelixClusterAdminTest extends TestBaseWithZookeeper { if (clusterAdmin2 != null) { clusterAdmin2.stop(); } + + zkHelixAdmin.dropCluster(CLUSTER_NAME); cleanupTestMetadata(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java index 8193884..c347219 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java @@ -52,8 +52,6 @@ // // private static Server server; // -// private static String ZK_ADDRESS = "localhost:2199"; -// // private static KylinConfig configA; // private static KylinConfig configB; // http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java index ca4fe39..020b2d0 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java +++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java @@ -18,12 +18,6 @@ package org.apache.kylin.rest.service; -import com.google.common.collect.Lists; -import org.I0Itec.zkclient.IDefaultNameSpace; -import org.I0Itec.zkclient.ZkClient; -import org.I0Itec.zkclient.ZkServer; -import org.apache.hadoop.fs.FileUtil; -import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeManager; @@ -32,42 +26,40 @@ import org.apache.kylin.invertedindex.IIManager; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.RealizationRegistry; -import org.apache.kylin.rest.helix.HelixClusterAdmin; import org.junit.*; import org.junit.runner.RunWith; import org.springframework.security.authentication.TestingAuthenticationToken; import org.springframework.security.core.Authentication; +import org.springframework.security.core.authority.AuthorityUtils; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.core.userdetails.User; -import org.springframework.security.core.userdetails.UserDetails; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import java.io.File; -import java.util.Arrays; -import java.util.List; - /** * @author xduo */ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = { "classpath:applicationContext.xml", "classpath:kylinSecurity.xml" }) +@ContextConfiguration(locations = {"classpath:applicationContext.xml", "classpath:kylinSecurity.xml"}) @ActiveProfiles("testing") -public class ServiceTestBase extends TestBaseWithZookeeper { +public class ServiceTestBase extends LocalFileMetadataTestCase { + + @BeforeClass + public static void setupResource() throws Exception { + staticCreateTestMetadata(); + Authentication authentication = new TestingAuthenticationToken(new User("ADMIN", "ADMIN", AuthorityUtils.createAuthorityList("ROLE_ADMIN")), "ADMIN", "ROLE_ADMIN"); + SecurityContextHolder.getContext().setAuthentication(authentication); + } + + @AfterClass + public static void tearDownResource() { + } @Before public void setup() throws Exception { this.createTestMetadata(); - UserService.UserGrantedAuthority userGrantedAuthority = new UserService.UserGrantedAuthority(); - userGrantedAuthority.setAuthority("ROLE_ADMIN"); - UserDetails user = new User("ADMIN", "skippped-ldap", Lists.newArrayList(userGrantedAuthority)); - Authentication authentication = new TestingAuthenticationToken(user, "ADMIN", "ROLE_ADMIN"); - SecurityContextHolder.getContext().setAuthentication(authentication); - KylinConfig kylinConfig = this.getTestConfig(); - kylinConfig.setRestAddress("localhost:7070"); - MetadataManager.clearCache(); CubeDescManager.clearCache(); CubeManager.clearCache(); http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java b/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java index 3182c16..e51e632 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java +++ b/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java @@ -25,9 +25,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.springframework.security.authentication.TestingAuthenticationToken; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContextHolder; import java.io.File; @@ -35,7 +32,7 @@ import java.io.File; */ public class TestBaseWithZookeeper extends LocalFileMetadataTestCase { protected static final String zkAddress = "localhost:2199"; - static ZkServer server; + static ZkServer server = null; static boolean zkStarted = false; @BeforeClass @@ -57,13 +54,13 @@ public class TestBaseWithZookeeper extends LocalFileMetadataTestCase { zkStarted = true; System.setProperty("kylin.zookeeper.address", zkAddress); } - } @AfterClass public static void tearDownResource() { - if (server == null) { + if (server != null) { server.shutdown(); + server = null; zkStarted = false; System.setProperty("kylin.zookeeper.address", ""); }