Repository: incubator-gearpump Updated Branches: refs/heads/master 956d52a35 -> cae9f9c7d
[GEARPUMP-254] fix gearpump on Yarn Be sure to do all of the following to help us incorporate your contribution quickly and easily: - [x] Make sure the commit message is formatted like: `[GEARPUMP-<Jira issue #>] Meaningful description of pull request` - [x] Make sure tests pass via `sbt clean test`. - [x] Make sure old documentation affected by the pull request has been updated and new documentation added for new functionality. Author: huafengw <[email protected]> Closes #128 from huafengw/fixYarn. Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/cae9f9c7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/cae9f9c7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/cae9f9c7 Branch: refs/heads/master Commit: cae9f9c7d3ea9e3b2cf173c0a2deac61964bc289 Parents: 956d52a Author: huafengw <[email protected]> Authored: Wed Jan 4 13:49:32 2017 +0800 Committer: manuzhang <[email protected]> Committed: Wed Jan 4 13:49:32 2017 +0800 ---------------------------------------------------------------------- .../experiments/yarn/appmaster/Command.scala | 2 +- .../experiments/yarn/client/LaunchCluster.scala | 4 +- .../yarn/appmaster/CommandSpec.scala | 95 -------------------- .../yarn/client/LaunchClusterSpec.scala | 2 +- project/BuildExperiments.scala | 2 +- project/BuildGearpump.scala | 2 +- 6 files changed, 6 insertions(+), 101 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cae9f9c7/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala ---------------------------------------------------------------------- diff --git a/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala b/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala index 711506a..fb482c4 100644 --- a/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala +++ b/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/appmaster/Command.scala @@ -48,7 +48,7 @@ abstract class AbstractCommand extends Command { : String = { val exe = config.getString(java) - s"$exe -cp ${classPath.mkString(":")}:" + + s"$exe -noverify -cp ${classPath.mkString(":")}:" + "$CLASSPATH " + properties.mkString(" ") + s" $mainClazz ${cliOpts.mkString(" ")} 2>&1 | /usr/bin/tee -a ${LOG_DIR_EXPANSION_VAR}/stderr" } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cae9f9c7/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala ---------------------------------------------------------------------- diff --git a/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala b/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala index 98cff61..267d588 100644 --- a/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala +++ b/experiments/yarn/src/main/scala/org/apache/gearpump/experiments/yarn/client/LaunchCluster.scala @@ -185,8 +185,8 @@ object LaunchCluster extends AkkaApp with ArgumentsParser { if (parsed.getBoolean(VERBOSE)) { LogUtil.verboseLogToConsole() } - val userName = inputAkkaConf.getString(CONTAINER_USER) - if (userName != null) { + if (inputAkkaConf.hasPath(CONTAINER_USER)) { + val userName = inputAkkaConf.getString(CONTAINER_USER) System.setProperty("HADOOP_USER_NAME", userName) } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cae9f9c7/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/appmaster/CommandSpec.scala ---------------------------------------------------------------------- diff --git a/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/appmaster/CommandSpec.scala b/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/appmaster/CommandSpec.scala deleted file mode 100644 index c4c5a65..0000000 --- a/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/appmaster/CommandSpec.scala +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.gearpump.experiments.yarn.appmaster - -import com.typesafe.config.ConfigFactory -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} - -import org.apache.gearpump.cluster.TestUtil -import org.apache.gearpump.transport.HostPort - -class CommandSpec extends FlatSpec with Matchers with BeforeAndAfterAll { - val config = ConfigFactory.parseString( - - """ - | - |gearpump { - | yarn { - | client { - | package -path = "/user/gearpump/gearpump.zip" - | } - | - | applicationmaster { - | ## Memory of YarnAppMaster - | command = "$JAVA_HOME/bin/java -Xmx512m" - | memory = "512" - | vcores = "1" - | queue = "default" - | } - | - | master { - | ## Memory of master daemon - | command = "$JAVA_HOME/bin/java -Xmx512m" - | memory = "512" - | vcores = "1" - | } - | - | worker { - | ## memory of worker daemon - | command = "$JAVA_HOME/bin/java -Xmx512m" - | containers = "4" - | ## This also contains all memory for child executors. - | memory = "4096" - | vcores = "1" - | } - | services { - | enabled = true - | } - | } - |} - """.stripMargin).withFallback(TestUtil.DEFAULT_CONFIG) - - "MasterCommand" should "create correct command line" in { - val version = "gearpump-0.1" - val master = MasterCommand(config, version, HostPort("127.0.0.1", 8080)) - - // scalastyle:off line.size.limit - val expected = "$JAVA_HOME/bin/java -Xmx512m -cp conf:pack/gearpump-0.1/conf:pack/gearpump-0.1/lib/daemon/*:pack/gearpump-0.1/lib/*:$CLASSPATH -Dgearpump.cluster.masters.0=127.0.0.1:8080 -Dgearpump.hostname=127.0.0.1 -Dgearpump.master-resource-manager-container-id={{CONTAINER_ID}} -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.1 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.log.application.dir=<LOG_DIR> org.apache.gearpump.cluster.main.Master -ip 127.0.0.1 -port 8080 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr" - // scalastyle:on line.size.limit - assert(master.get == expected) - } - - "WorkerCommand" should "create correct command line" in { - val version = "gearpump-0.1" - val worker = WorkerCommand(config, version, HostPort("127.0.0.1", 8080), "worker-machine") - // scalastyle:off line.size.limit - val expected = "$JAVA_HOME/bin/java -Xmx512m -cp conf:pack/gearpump-0.1/conf:pack/gearpump-0.1/lib/daemon/*:pack/gearpump-0.1/lib/*:$CLASSPATH -Dgearpump.cluster.masters.0=127.0.0.1:8080 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.worker-resource-manager-container-id={{CONTAINER_ID}} -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.1 -Dgearpump.log.application.dir=<LOG_DIR> -Dgearpump.hostname=worker-machine org.apache.gearpump.cluster.main.Worker 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr" - // scalastyle:on line.size.limit - assert(worker.get == expected) - } - - "AppMasterCommand" should "create correct command line" in { - val version = "gearpump-0.1" - val appmaster = AppMasterCommand(config, version, Array("arg1", "arg2", "arg3")) - // scalastyle:off line.size.limit - val expected = "$JAVA_HOME/bin/java -Xmx512m -cp conf:pack/gearpump-0.1/conf:pack/gearpump-0.1/dashboard:pack/gearpump-0.1/lib/*:pack/gearpump-0.1/lib/daemon/*:pack/gearpump-0.1/lib/services/*:pack/gearpump-0.1/lib/yarn/*:$CLASSPATH -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.1 -Dgearpump.binary-version-with-scala-version=gearpump-0.1 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.log.application.dir=<LOG_DIR> -Dgearpump.hostname={{NM_HOST}} org.apache.gearpump.experiments.yarn.appmaster.YarnAppMaster arg1 arg2 arg3 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr" - // scalastyle:on line.size.limit - assert(appmaster.get == expected) - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cae9f9c7/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala ---------------------------------------------------------------------- diff --git a/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala b/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala index c0a4ee2..3f2f9cb 100644 --- a/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala +++ b/experiments/yarn/src/test/scala/org/apache/gearpump/experiments/yarn/client/LaunchClusterSpec.scala @@ -186,7 +186,7 @@ class LaunchClusterSpec extends FlatSpec with Matchers with BeforeAndAfterAll { // val workerResources = ArgumentCaptor.forClass(classOf[List[Resource]]) // scalastyle:off line.size.limit - val expectedCommand = "$JAVA_HOME/bin/java -Xmx512m -cp conf:pack/gearpump-0.2/conf:pack/gearpump-0.2/dashboard:pack/gearpump-0.2/lib/*:pack/gearpump-0.2/lib/daemon/*:pack/gearpump-0.2/lib/services/*:pack/gearpump-0.2/lib/yarn/*:$CLASSPATH -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.2 -Dgearpump.binary-version-with-scala-version=gearpump-0.2 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.log.application.dir=<LOG_DIR> -Dgearpump.hostname={{NM_HOST}} org.apache.gearpump.experiments.yarn.appmaster.YarnAppMaster -conf /root/.gearpump_application_0_0000/conf/ -package gearpump.zip 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr" + val expectedCommand = "$JAVA_HOME/bin/java -Xmx512m -noverify -cp conf:pack/gearpump-0.2/conf:pack/gearpump-0.2/dashboard:pack/gearpump-0.2/lib/*:pack/gearpump-0.2/lib/daemon/*:pack/gearpump-0.2/lib/services/*:pack/gearpump-0.2/lib/yarn/*:$CLASSPATH -Dgearpump.home={{LOCAL_DIRS}}/{{CONTAINER_ID}}/pack/gearpump-0.2 -Dgearpump.binary-version-with-scala-version=gearpump-0.2 -Dgearpump.log.daemon.dir=<LOG_DIR> -Dgearpump.log.application.dir=<LOG_DIR> -Dgearpump.hostname={{NM_HOST}} org.apache.gearpump.experiments.yarn.appmaster.YarnAppMaster -conf /root/.gearpump_application_0_0000/conf/ -package gearpump.zip 2>&1 | /usr/bin/tee -a <LOG_DIR>/stderr" // scalastyle:on line.size.limit verify(yarnClient).submit("gearpump", appId, expectedCommand, Resource.newInstance(512, 1), "default", http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cae9f9c7/project/BuildExperiments.scala ---------------------------------------------------------------------- diff --git a/project/BuildExperiments.scala b/project/BuildExperiments.scala index 3bebf22..550b407 100644 --- a/project/BuildExperiments.scala +++ b/project/BuildExperiments.scala @@ -115,6 +115,6 @@ object BuildExperiments extends sbt.Build { id = "gearpump-experimental-cgroup", base = file("experiments/cgroup"), settings = commonSettings ++ noPublish) - .dependsOn (core % "test->test; compile->compile") + .dependsOn (core % "provided") .disablePlugins(sbtassembly.AssemblyPlugin) } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cae9f9c7/project/BuildGearpump.scala ---------------------------------------------------------------------- diff --git a/project/BuildGearpump.scala b/project/BuildGearpump.scala index 793d903..f0c9517 100644 --- a/project/BuildGearpump.scala +++ b/project/BuildGearpump.scala @@ -221,7 +221,7 @@ object BuildGearpump extends sbt.Build { "org.apache.hadoop" % "hadoop-common" % hadoopVersion ) ) - ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin) + ).dependsOn(core % "provided").disablePlugins(sbtassembly.AssemblyPlugin) private def changeShadedDeps(toExclude: Set[String], toInclude: List[xml.Node], node: xml.Node): xml.Node = {
