Dear list,
SBT compiles fine, but when I do the following:
sbt/sbt gen-idea
import project as SBT project to IDEA 13.1
Make Project
and these errors show up:
Error:(28, 8) object FileContext is not a member of package
org.apache.hadoop.fs
import org.apache.hadoop.fs.{FileContext, FileStatus, FileSystem, Path,
FileUtil}
^
Error:(31, 8) object Master is not a member of package
org.apache.hadoop.mapred
import org.apache.hadoop.mapred.Master
^
Error:(34, 26) object yarn is not a member of package org.apache.hadoop
import org.apache.hadoop.yarn.api._
^
Error:(35, 26) object yarn is not a member of package org.apache.hadoop
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
^
Error:(36, 26) object yarn is not a member of package org.apache.hadoop
import org.apache.hadoop.yarn.api.protocolrecords._
^
Error:(37, 26) object yarn is not a member of package org.apache.hadoop
import org.apache.hadoop.yarn.api.records._
^
Error:(38, 26) object yarn is not a member of package org.apache.hadoop
import org.apache.hadoop.yarn.client.YarnClientImpl
^
Error:(39, 26) object yarn is not a member of package org.apache.hadoop
import org.apache.hadoop.yarn.conf.YarnConfiguration
^
Error:(40, 26) object yarn is not a member of package org.apache.hadoop
import org.apache.hadoop.yarn.ipc.YarnRPC
^
Error:(41, 26) object yarn is not a member of package org.apache.hadoop
import org.apache.hadoop.yarn.util.{Apps, Records}
^
Error:(49, 11) not found: type YarnClientImpl
extends YarnClientImpl with Logging {
^
Error:(48, 20) not found: type ClientArguments
class Client(args: ClientArguments, conf: Configuration, sparkConf:
SparkConf)
^
Error:(51, 18) not found: type ClientArguments
def this(args: ClientArguments, sparkConf: SparkConf) =
^
Error:(54, 18) not found: type ClientArguments
def this(args: ClientArguments) = this(args, new SparkConf())
^
Error:(56, 12) not found: type YarnRPC
var rpc: YarnRPC = YarnRPC.create(conf)
^
Error:(56, 22) not found: value YarnRPC
var rpc: YarnRPC = YarnRPC.create(conf)
^
Error:(57, 17) not found: type YarnConfiguration
val yarnConf: YarnConfiguration = new YarnConfiguration(conf)
^
Error:(57, 41) not found: type YarnConfiguration
val yarnConf: YarnConfiguration = new YarnConfiguration(conf)
^
Error:(58, 59) value getCredentials is not a member of
org.apache.hadoop.security.UserGroupInformation
val credentials = UserGroupInformation.getCurrentUser().getCredentials()
^
Error:(60, 34) not found: type ClientDistributedCacheManager
private val distCacheMgr = new ClientDistributedCacheManager()
^
Error:(72, 5) not found: value init
init(yarnConf)
^
Error:(73, 5) not found: value start
start()
^
Error:(76, 24) value getNewApplication is not a member of
org.apache.spark.Logging
val newApp = super.getNewApplication()
^
Error:(137, 35) not found: type GetNewApplicationResponse
def verifyClusterResources(app: GetNewApplicationResponse) = {
^
Error:(156, 65) not found: type ApplicationSubmissionContext
def createApplicationSubmissionContext(appId: ApplicationId):
ApplicationSubmissionContext = {
^
Error:(156, 49) not found: type ApplicationId
def createApplicationSubmissionContext(appId: ApplicationId):
ApplicationSubmissionContext = {
^
Error:(118, 31) not found: type ApplicationId
def getAppStagingDir(appId: ApplicationId): String = {
^
Error:(224, 69) not found: type LocalResource
def prepareLocalResources(appStagingDir: String): HashMap[String,
LocalResource] = {
^
Error:(307, 39) not found: type LocalResource
localResources: HashMap[String, LocalResource],
^
Error:(343, 38) not found: type ContainerLaunchContext
env: HashMap[String, String]): ContainerLaunchContext = {
^
Error:(341, 15) not found: type GetNewApplicationResponse
newApp: GetNewApplicationResponse,
^
Error:(342, 39) not found: type LocalResource
localResources: HashMap[String, LocalResource],
^
Error:(426, 11) value submitApplication is not a member of
org.apache.spark.Logging
super.submitApplication(appContext)
^
Error:(423, 29) not found: type ApplicationSubmissionContext
def submitApp(appContext: ApplicationSubmissionContext) = {
^
Error:(429, 33) not found: type ApplicationId
def monitorApplication(appId: ApplicationId): Boolean = {
^
Error:(123, 25) not found: type YarnClusterMetrics
val clusterMetrics: YarnClusterMetrics = super.getYarnClusterMetrics
^
Error:(123, 52) value getYarnClusterMetrics is not a member of
org.apache.spark.Logging
val clusterMetrics: YarnClusterMetrics = super.getYarnClusterMetrics
^
Error:(127, 20) not found: type QueueInfo
val queueInfo: QueueInfo = super.getQueueInfo(args.amQueue)
^
Error:(127, 38) value getQueueInfo is not a member of
org.apache.spark.Logging
val queueInfo: QueueInfo = super.getQueueInfo(args.amQueue)
^
Error:(158, 22) not found: value Records
val appContext =
Records.newRecord(classOf[ApplicationSubmissionContext])
^
Error:(219, 14) not found: value FileContext
val fc = FileContext.getFileContext(qualPath.toUri(), conf)
^
Error:(230, 29) not found: value Master
val delegTokenRenewer = Master.getMasterPrincipal(conf)
^
Error:(242, 13) value addDelegationTokens is not a member of
org.apache.hadoop.fs.FileSystem
dstFs.addDelegationTokens(delegTokenRenewer, credentials)
^
Error:(244, 42) not found: type LocalResource
val localResources = HashMap[String, LocalResource]()
^
Error:(302, 43) value addCredentials is not a member of
org.apache.hadoop.security.UserGroupInformation
UserGroupInformation.getCurrentUser().addCredentials(credentials)
^
Error:(323, 5) not found: value Apps
Apps.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV"))
^
Error:(330, 36) not found: type ClientArguments
def userArgsToString(clientArgs: ClientArguments): String = {
^
Error:(345, 23) not found: value Records
val amContainer = Records.newRecord(classOf[ContainerLaunchContext])
^
Error:(363, 16) not found: value Environment
new Path(Environment.PWD.$(),
YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR) + " "
^
Error:(392, 21) not found: value Environment
javaCommand = Environment.JAVA_HOME.$() + "/bin/java"
^
Error:(405, 16) not found: value ApplicationConstants
" 1> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
^
Error:(410, 22) not found: value Records
val capability =
Records.newRecord(classOf[Resource]).asInstanceOf[Resource]
^
Error:(410, 72) not found: type Resource
val capability =
Records.newRecord(classOf[Resource]).asInstanceOf[Resource]
^
Error:(434, 26) value getApplicationReport is not a member of
org.apache.spark.Logging
val report = super.getApplicationReport(appId)
^
Error:(474, 20) not found: type ClientArguments
val args = new ClientArguments(argStrings, sparkConf)
^
Error:(481, 31) not found: value YarnConfiguration
for (c <-
conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) {
^
Error:(487, 5) not found: value Apps
Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$())
^
Error:(490, 7) not found: value Apps
Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
^
Error:(496, 7) not found: value Apps
Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
^
Error:(499, 5) not found: value Apps
Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
^
Error:(504, 7) not found: value Apps
Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
^
Error:(507, 5) not found: value Apps
Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
^
Any idea what's causing them, and maybe I am not using the best practice to
import Spark to IDE?
I would appreciate any suggestion on the best practice to import Spark to
any IDE.
Thank you
-Mo
