[ https://issues.apache.org/jira/browse/SPARK-2815?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14084287#comment-14084287 ]
pengyanhong commented on SPARK-2815: ------------------------------------ I changed the YarnAllocationHandler.scala file as below: import org.apache.hadoop.yarn.api.records,ApplicationAttemptId val amResp = allocateExecutorResources(executorsToRequest) then compile successfully and it can work on YARN cluster, but i am not sure whether there are potential problems. > Compilation failed upon the hadoop version 2.0.0-cdh4.5.0 > --------------------------------------------------------- > > Key: SPARK-2815 > URL: https://issues.apache.org/jira/browse/SPARK-2815 > Project: Spark > Issue Type: Bug > Components: Build > Affects Versions: 1.1.0 > Reporter: pengyanhong > Assignee: Guoqiang Li > Priority: Blocker > > compile fail via SPARK_HADOOP_VERSION=2.0.0-cdh4.5.0 SPARK_YARN=true > SPARK_HIVE=true sbt/sbt assembly, finally get error message : [error] > (yarn-stable/compile:compile) Compilation failed, the following is the detail > error on console: > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:26: > object api is not a member of package org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.api.YarnClient > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:40: > not found: value YarnClient > [error] val yarnClient = YarnClient.createYarnClient > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:32: > object api is not a member of package org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.api.AMRMClient > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:33: > object api is not a member of package org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:36: > object util is not a member of package org.apache.hadoop.yarn.webapp > [error] import org.apache.hadoop.yarn.webapp.util.WebAppUtils > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:64: > value RM_AM_MAX_ATTEMPTS is not a member of object > org.apache.hadoop.yarn.conf.YarnConfiguration > [error] YarnConfiguration.RM_AM_MAX_ATTEMPTS, > YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS) > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:66: > not found: type AMRMClient > [error] private var amClient: AMRMClient[ContainerRequest] = _ > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:92: > not found: value AMRMClient > [error] amClient = AMRMClient.createAMRMClient() > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:137: > not found: value WebAppUtils > [error] val proxy = WebAppUtils.getProxyHostAndPort(conf) > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:40: > object api is not a member of package org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.api.AMRMClient > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:618: > not found: type AMRMClient > [error] amClient: AMRMClient[ContainerRequest], > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:596: > not found: type AMRMClient > [error] amClient: AMRMClient[ContainerRequest], > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:577: > not found: type AMRMClient > [error] amClient: AMRMClient[ContainerRequest], > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:410: > value CONTAINER_ID is not a member of object > org.apache.hadoop.yarn.api.ApplicationConstants.Environment > [error] val containerIdString = > System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name()) > [error] > ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:128: > value setTokens is not a member of > org.apache.hadoop.yarn.api.records.ContainerLaunchContext > [error] amContainer.setTokens(ByteBuffer.wrap(dob.getData())) > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala:33: > object api is not a member of package org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.api.AMRMClient > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala:34: > object api is not a member of package org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala:36: > object util is not a member of package org.apache.hadoop.yarn.webapp > [error] import org.apache.hadoop.yarn.webapp.util.WebAppUtils > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala:61: > not found: type AMRMClient > [error] private var amClient: AMRMClient[ContainerRequest] = _ > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala:102: > not found: value AMRMClient > [error] amClient = AMRMClient.createAMRMClient() > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala:166: > not found: value WebAppUtils > [error] val proxy = WebAppUtils.getProxyHostAndPort(conf) > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala:31: > object ProtoUtils is not a member of package > org.apache.hadoop.yarn.api.records.impl.pb > [error] import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala:33: > object api is not a member of package org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.api.NMClient > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala:53: > not found: type NMClient > [error] var nmClient: NMClient = _ > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala:59: > not found: value NMClient > [error] nmClient = NMClient.createNMClient() > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala:79: > value setTokens is not a member of > org.apache.hadoop.yarn.api.records.ContainerLaunchContext > [error] ctx.setTokens(ByteBuffer.wrap(dob.getData())) > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:35: > object ApplicationMasterProtocol is not a member of package > org.apache.hadoop.yarn.api > [error] import org.apache.hadoop.yarn.api.ApplicationMasterProtocol > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:41: > object api is not a member of package org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:65: > not found: type AMRMClient > [error] val amClient: AMRMClient[ContainerRequest], > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:389: > not found: type ContainerRequest > [error] ): ArrayBuffer[ContainerRequest] = { > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:388: > not found: type ContainerRequest > [error] hostContainers: ArrayBuffer[ContainerRequest] > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:405: > not found: type ContainerRequest > [error] val requestedContainers = new > ArrayBuffer[ContainerRequest](rackToCounts.size) > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:434: > not found: type ContainerRequest > [error] val containerRequests: List[ContainerRequest] = > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:508: > not found: type ContainerRequest > [error] ): ArrayBuffer[ContainerRequest] = { > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:446: > not found: type ContainerRequest > [error] val hostContainerRequests = new > ArrayBuffer[ContainerRequest](preferredHostToCount.size) > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:458: > not found: type ContainerRequest > [error] val rackContainerRequests: List[ContainerRequest] = > createRackResourceRequests( > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:467: > not found: type ContainerRequest > [error] val containerRequestBuffer = new > ArrayBuffer[ContainerRequest]( > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:542: > not found: type ContainerRequest > [error] ): ArrayBuffer[ContainerRequest] = { > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:545: > value newInstance is not a member of object > org.apache.hadoop.yarn.api.records.Resource > [error] val resource = Resource.newInstance(memoryRequest, executorCores) > [error] ^ > [error] > /Users/pengyanhong/git/spark/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:550: > not found: type ContainerRequest > [error] val requests = new ArrayBuffer[ContainerRequest]() > [error] ^ > [error] 40 errors found -- This message was sent by Atlassian JIRA (v6.2#6252) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org