I am trying to setup Spark with YARN 2.2.0. My Hadoop is plain Hadoop from Apache Hadoop website. When I SBT build against 2.2.0 it fails. While it compiles with a lot of warnings when I try against Hadoop 2.0.5-alpha.
How can I compile Spark against YARN 2.2.0. There is a related thread here: https://groups.google.com/forum/#!topic/spark-users/8Gm6ByvdNME It did help. Also, I am a novice to SBT. Here is the error log: [root@ip-10-110-241-90 spark-0.8.0-incubating]# SPARK_HADOOP_VERSION=2.2.0 SPARK_YARN=true sbt/sbt clean assembly [info] Loading project definition from /tmp/spark/spark-0.8.0-incubating/project/project [-- snip --] [warn] jobCommitter.cleanupJob(jobTaskContext) [warn] ^ [warn] /tmp/spark/spark-0.8.0-incubating/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala:98: constructor Job in class Job is deprecated: see corresponding Javadoc for more information. [warn] val job = new Job(conf) [warn] ^ [warn] 9 warnings found [warn] Note: Some input files use unchecked or unsafe operations. [warn] Note: Recompile with -Xlint:unchecked for details. [info] Compiling 8 Scala sources to /tmp/spark/spark-0.8.0-incubating/yarn/target/scala-2.9.3/classes... [info] Compiling 50 Scala sources to /tmp/spark/spark-0.8.0-incubating/streaming/target/scala-2.9.3/classes... [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:42: not found: type AMRMProtocol [error] private var resourceManager: AMRMProtocol = null [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:126: not found: type AMRMProtocol [error] private def registerWithResourceManager(): AMRMProtocol = { [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:119: value AM_CONTAINER_ID_ENV is not a member of object org.apache.hadoop.yarn.api.ApplicationConstants [error] val containerIdString = envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:131: not found: type AMRMProtocol [error] return rpc.getProxy(classOf[AMRMProtocol], rmAddress, conf).asInstanceOf[AMRMProtocol] [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:131: not found: type AMRMProtocol [error] return rpc.getProxy(classOf[AMRMProtocol], rmAddress, conf).asInstanceOf[AMRMProtocol] [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:138: value setApplicationAttemptId is not a member of org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest [error] appMasterRequest.setApplicationAttemptId(appAttemptId) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:30: AMRMProtocol is not a member of org.apache.hadoop.yarn.api [error] import org.apache.hadoop.yarn.api.AMRMProtocol [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:504: not found: type AMRMProtocol [error] resourceManager: AMRMProtocol, appAttemptId: ApplicationAttemptId, [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:24: AMResponse is not a member of org.apache.hadoop.yarn.api.records [error] import org.apache.hadoop.yarn.api.records.{AMResponse, ApplicationAttemptId, ContainerId, Priority, Resource, ResourceRequest, ContainerStatus, Container} [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:493: not found: type AMRMProtocol [error] resourceManager: AMRMProtocol, appAttemptId: ApplicationAttemptId, [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:485: not found: type AMRMProtocol [error] resourceManager: AMRMProtocol, appAttemptId: ApplicationAttemptId, [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:325: value setAppAttemptId is not a member of org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest [error] finishReq.setAppAttemptId(appAttemptId) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:326: value setFinishApplicationStatus is not a member of org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest [error] finishReq.setFinishApplicationStatus(status) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:31: YarnClientImpl is not a member of org.apache.hadoop.yarn.client [error] import org.apache.hadoop.yarn.client.YarnClientImpl [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:42: not found: type YarnClientImpl [error] class Client(conf: Configuration, args: ClientArguments) extends YarnClientImpl with Logging { [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:51: not found: value init [error] init(yarnConf) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:52: not found: value start [error] start() [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:55: value getNewApplication is not a member of AnyRef with org.apache.spark.Logging with ScalaObject [error] val newApp = super.getNewApplication() [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:66: value setUser is not a member of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext [error] appContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName()) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:286: value submitApplication is not a member of AnyRef with org.apache.spark.Logging with ScalaObject [error] super.submitApplication(appContext) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:76: value getYarnClusterMetrics is not a member of AnyRef with org.apache.spark.Logging with ScalaObject [error] val clusterMetrics: YarnClusterMetrics = super.getYarnClusterMetrics [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:79: value getQueueInfo is not a member of AnyRef with org.apache.spark.Logging with ScalaObject [error] val queueInfo: QueueInfo = super.getQueueInfo(args.amQueue) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:216: value getMinimumResourceCapability is not a member of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse [error] val minResMemory: Int = newApp.getMinimumResourceCapability().getMemory() [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:273: value setResource is not a member of org.apache.hadoop.yarn.api.records.ContainerLaunchContext [error] amContainer.setResource(capability) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:278: value setContainerTokens is not a member of org.apache.hadoop.yarn.api.records.ContainerLaunchContext [error] amContainer.setContainerTokens(ByteBuffer.wrap(dob.getData())) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:292: value getApplicationReport is not a member of AnyRef with org.apache.spark.Logging with ScalaObject [error] val report = super.getApplicationReport(appId) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:34: ProtoUtils is not a member of org.apache.hadoop.yarn.util [error] import org.apache.hadoop.yarn.util.{Apps, ConverterUtils, Records, ProtoUtils} [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:48: not found: type ContainerManager [error] var cm: ContainerManager = null [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:202: not found: type ContainerManager [error] def connectToCM: ContainerManager = { [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:63: value setContainerId is not a member of org.apache.hadoop.yarn.api.records.ContainerLaunchContext [error] ctx.setContainerId(container.getId()) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:64: value setResource is not a member of org.apache.hadoop.yarn.api.records.ContainerLaunchContext [error] ctx.setResource(container.getResource()) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:103: value setUser is not a member of org.apache.hadoop.yarn.api.records.ContainerLaunchContext [error] ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:108: value setContainerTokens is not a member of org.apache.hadoop.yarn.api.records.ContainerLaunchContext [error] ctx.setContainerTokens(ByteBuffer.wrap(dob.getData())) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:212: not found: value ProtoUtils [error] user.addToken(ProtoUtils.convertFromProtoFormat(containerToken, cmAddress)) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:216: not found: type ContainerManager [error] .doAs(new PrivilegedExceptionAction[ContainerManager] { [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:49: not found: type AMRMProtocol [error] private[yarn] class YarnAllocationHandler(val conf: Configuration, val resourceManager: AMRMProtocol, [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:87: value getAMResponse is not a member of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse [error] val amResp = allocateWorkerResources(workersToRequest).getAMResponse [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:296: value getHostName is not a member of org.apache.hadoop.yarn.api.records.ResourceRequest [error] val candidateHost = container.getHostName [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:374: value setApplicationAttemptId is not a member of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest [error] req.setApplicationAttemptId(appAttemptId) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:376: value addAllAsks is not a member of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest [error] req.addAllAsks(resourceRequests) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:379: value addAllReleases is not a member of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest [error] req.addAllReleases(releasedContainerList) [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:391: value getHostName is not a member of org.apache.hadoop.yarn.api.records.ResourceRequest [error] logInfo("rsrcRequest ... host : " + req.getHostName + ", numContainers : " + req.getNumContainers + [error] ^ [error] /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:441: value setHostName is not a member of org.apache.hadoop.yarn.api.records.ResourceRequest [error] rsrcRequest.setHostName(hostname) [error] ^ [error] 43 errors found [-- snip --] [warn] Merging 'org/yaml/snakeyaml/constructor/SafeConstructor$ConstructYamlBool.class' with strategy 'first' [warn] Merging 'org/yaml/snakeyaml/emitter/Emitter$ExpectStreamStart.class' with strategy 'first' [warn] Strategy 'concat' was applied to 2 files [warn] Strategy 'discard' was applied to 2 files [warn] Strategy 'first' was applied to 794 files [info] Checking every *.class/*.jar file's SHA-1. [info] SHA-1: deebf2bd4f022965649cfe78d51ff1c8780c92a2 [info] Packaging /tmp/spark/spark-0.8.0-incubating/examples/target/scala-2.9.3/spark-examples-assembly-0.8.0-incubating.jar ... [info] Done packaging. [error] (yarn/compile:compile) Compilation failed [error] Total time: 668 s, completed 12 Dec, 2013 10:15:25 AM
