roadan commented on a change in pull request #44: Amaterasu 45
URL: https://github.com/apache/incubator-amaterasu/pull/44#discussion_r261847641
##########
File path:
frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
##########
@@ -4,49 +4,42 @@ import java.net.URLEncoder
import org.apache.amaterasu.common.configuration.ClusterConfig
import org.apache.amaterasu.common.dataobjects.ActionData
+import
org.apache.amaterasu.frameworks.python.dispatcher.runners.providers.PythonRunnerProviderBase
import org.apache.amaterasu.leader.common.utilities.DataLoader
import org.apache.amaterasu.sdk.frameworks.RunnerSetupProvider
import org.apache.hadoop.yarn.api.ApplicationConstants
-class PySparkRunnerProvider extends RunnerSetupProvider {
+class PySparkRunnerProvider(val env: String, val conf: ClusterConfig) extends
PythonRunnerProviderBase(env, conf) {
- private var conf: ClusterConfig = _
private val libPath = System.getProperty("java.library.path")
- override def getCommand(jobId: String, actionData: ActionData, env: String,
executorId: String, callbackAddress: String): String = conf.mode match {
- case "mesos" =>
- s"env AMA_NODE=${sys.env("AMA_NODE")} env
MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so env
SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${conf.Webserver.Port}/dist/spark-${conf.Webserver.sparkVersion}.tgz
" +
- s"java -cp
executor-${conf.version}-all.jar:spark-runner-${conf.version}-all.jar:spark-runtime-${conf.version}.jar:spark-${conf.Webserver.sparkVersion}/jars/*
" +
- s"-Dscala.usejavacp=true -Djava.library.path=$libPath
org.apache.amaterasu.executor.mesos.executors.MesosActionsExecutor $jobId
${conf.master} ${actionData.getName}.stripMargin"
- case "yarn" => "/bin/bash ./miniconda.sh -b -p $PWD/miniconda && " +
- s"/bin/bash spark/bin/load-spark-env.sh && " +
- s"java -cp
spark/jars/*:executor.jar:spark-runner.jar:spark-runtime.jar:spark/conf/:${conf.YARN.hadoopHomeDir}/conf/
" +
- "-Xmx2G " +
- "-Dscala.usejavacp=true " +
- "-Dhdp.version=2.6.1.0-129 " +
- "org.apache.amaterasu.executor.yarn.executors.ActionsExecutorLauncher " +
- s"'$jobId' '${conf.master}' '${actionData.getName}'
'${URLEncoder.encode(DataLoader.getTaskDataString(actionData, env), "UTF-8")}'
'${URLEncoder.encode(DataLoader.getExecutorDataString(env, conf), "UTF-8")}'
'$executorId' '$callbackAddress' " +
- s"1> ${ApplicationConstants.LOG_DIR_EXPANSION_VAR}/stdout " +
- s"2> ${ApplicationConstants.LOG_DIR_EXPANSION_VAR}/stderr "
- case _ => ""
+ override def getCommand(jobId: String, actionData: ActionData, env: String,
executorId: String, callbackAddress: String): String = {
+ val command = super.getCommand(jobId: String, actionData: ActionData, env:
String, executorId: String, callbackAddress: String)
+ command + conf.mode match {
+ case "mesos" =>
+ s" && env AMA_NODE=${sys.env("AMA_NODE")} env
MESOS_NATIVE_JAVA_LIBRARY=${conf.mesos.libPath}" +
+ s" && python3 ${actionData.getSrc}"
+ case "yarn" => s" && /bin/bash spark/bin/load-spark-env.sh" +
+ s" && python3 ${actionData.getSrc}"
Review comment:
this should be spark-submit, not using python3
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
