I let IntelliJ read the Maven build directly and that works fine.
--
Sean Owen | Director, Data Science | London
On Wed, Apr 9, 2014 at 6:14 AM, Dong Mo <monted...@gmail.com> wrote:
> Dear list,
>
> SBT compiles fine, but when I do the following:
> sbt/sbt gen-idea
> import project as SBT project to IDEA 13.1
> Make Project
> and these errors show up:
>
> Error:(28, 8) object FileContext is not a member of package
> org.apache.hadoop.fs
> import org.apache.hadoop.fs.{FileContext, FileStatus, FileSystem, Path,
> FileUtil}
> ^
> Error:(31, 8) object Master is not a member of package
> org.apache.hadoop.mapred
> import org.apache.hadoop.mapred.Master
> ^
> Error:(34, 26) object yarn is not a member of package org.apache.hadoop
> import org.apache.hadoop.yarn.api._
> ^
> Error:(35, 26) object yarn is not a member of package org.apache.hadoop
> import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
> ^
> Error:(36, 26) object yarn is not a member of package org.apache.hadoop
> import org.apache.hadoop.yarn.api.protocolrecords._
> ^
> Error:(37, 26) object yarn is not a member of package org.apache.hadoop
> import org.apache.hadoop.yarn.api.records._
> ^
> Error:(38, 26) object yarn is not a member of package org.apache.hadoop
> import org.apache.hadoop.yarn.client.YarnClientImpl
> ^
> Error:(39, 26) object yarn is not a member of package org.apache.hadoop
> import org.apache.hadoop.yarn.conf.YarnConfiguration
> ^
> Error:(40, 26) object yarn is not a member of package org.apache.hadoop
> import org.apache.hadoop.yarn.ipc.YarnRPC
> ^
> Error:(41, 26) object yarn is not a member of package org.apache.hadoop
> import org.apache.hadoop.yarn.util.{Apps, Records}
> ^
> Error:(49, 11) not found: type YarnClientImpl
> extends YarnClientImpl with Logging {
> ^
> Error:(48, 20) not found: type ClientArguments
> class Client(args: ClientArguments, conf: Configuration, sparkConf:
> SparkConf)
> ^
> Error:(51, 18) not found: type ClientArguments
> def this(args: ClientArguments, sparkConf: SparkConf) =
> ^
> Error:(54, 18) not found: type ClientArguments
> def this(args: ClientArguments) = this(args, new SparkConf())
> ^
> Error:(56, 12) not found: type YarnRPC
> var rpc: YarnRPC = YarnRPC.create(conf)
> ^
> Error:(56, 22) not found: value YarnRPC
> var rpc: YarnRPC = YarnRPC.create(conf)
> ^
> Error:(57, 17) not found: type YarnConfiguration
> val yarnConf: YarnConfiguration = new YarnConfiguration(conf)
> ^
> Error:(57, 41) not found: type YarnConfiguration
> val yarnConf: YarnConfiguration = new YarnConfiguration(conf)
> ^
> Error:(58, 59) value getCredentials is not a member of
> org.apache.hadoop.security.UserGroupInformation
> val credentials = UserGroupInformation.getCurrentUser().getCredentials()
> ^
> Error:(60, 34) not found: type ClientDistributedCacheManager
> private val distCacheMgr = new ClientDistributedCacheManager()
> ^
> Error:(72, 5) not found: value init
> init(yarnConf)
> ^
> Error:(73, 5) not found: value start
> start()
> ^
> Error:(76, 24) value getNewApplication is not a member of
> org.apache.spark.Logging
> val newApp = super.getNewApplication()
> ^
> Error:(137, 35) not found: type GetNewApplicationResponse
> def verifyClusterResources(app: GetNewApplicationResponse) = {
> ^
> Error:(156, 65) not found: type ApplicationSubmissionContext
> def createApplicationSubmissionContext(appId: ApplicationId):
> ApplicationSubmissionContext = {
> ^
> Error:(156, 49) not found: type ApplicationId
> def createApplicationSubmissionContext(appId: ApplicationId):
> ApplicationSubmissionContext = {
> ^
> Error:(118, 31) not found: type ApplicationId
> def getAppStagingDir(appId: ApplicationId): String = {
> ^
> Error:(224, 69) not found: type LocalResource
> def prepareLocalResources(appStagingDir: String): HashMap[String,
> LocalResource] = {
> ^
> Error:(307, 39) not found: type LocalResource
> localResources: HashMap[String, LocalResource],
> ^
> Error:(343, 38) not found: type ContainerLaunchContext
> env: HashMap[String, String]): ContainerLaunchContext = {
> ^
> Error:(341, 15) not found: type GetNewApplicationResponse
> newApp: GetNewApplicationResponse,
> ^
> Error:(342, 39) not found: type LocalResource
> localResources: HashMap[String, LocalResource],
> ^
> Error:(426, 11) value submitApplication is not a member of
> org.apache.spark.Logging
> super.submitApplication(appContext)
> ^
> Error:(423, 29) not found: type ApplicationSubmissionContext
> def submitApp(appContext: ApplicationSubmissionContext) = {
> ^
> Error:(429, 33) not found: type ApplicationId
> def monitorApplication(appId: ApplicationId): Boolean = {
> ^
> Error:(123, 25) not found: type YarnClusterMetrics
> val clusterMetrics: YarnClusterMetrics = super.getYarnClusterMetrics
> ^
> Error:(123, 52) value getYarnClusterMetrics is not a member of
> org.apache.spark.Logging
> val clusterMetrics: YarnClusterMetrics = super.getYarnClusterMetrics
> ^
> Error:(127, 20) not found: type QueueInfo
> val queueInfo: QueueInfo = super.getQueueInfo(args.amQueue)
> ^
> Error:(127, 38) value getQueueInfo is not a member of
> org.apache.spark.Logging
> val queueInfo: QueueInfo = super.getQueueInfo(args.amQueue)
> ^
> Error:(158, 22) not found: value Records
> val appContext =
> Records.newRecord(classOf[ApplicationSubmissionContext])
> ^
> Error:(219, 14) not found: value FileContext
> val fc = FileContext.getFileContext(qualPath.toUri(), conf)
> ^
> Error:(230, 29) not found: value Master
> val delegTokenRenewer = Master.getMasterPrincipal(conf)
> ^
> Error:(242, 13) value addDelegationTokens is not a member of
> org.apache.hadoop.fs.FileSystem
> dstFs.addDelegationTokens(delegTokenRenewer, credentials)
> ^
> Error:(244, 42) not found: type LocalResource
> val localResources = HashMap[String, LocalResource]()
> ^
> Error:(302, 43) value addCredentials is not a member of
> org.apache.hadoop.security.UserGroupInformation
> UserGroupInformation.getCurrentUser().addCredentials(credentials)
> ^
> Error:(323, 5) not found: value Apps
> Apps.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV"))
> ^
> Error:(330, 36) not found: type ClientArguments
> def userArgsToString(clientArgs: ClientArguments): String = {
> ^
> Error:(345, 23) not found: value Records
> val amContainer = Records.newRecord(classOf[ContainerLaunchContext])
> ^
> Error:(363, 16) not found: value Environment
> new Path(Environment.PWD.$(),
> YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR) + " "
> ^
> Error:(392, 21) not found: value Environment
> javaCommand = Environment.JAVA_HOME.$() + "/bin/java"
> ^
> Error:(405, 16) not found: value ApplicationConstants
> " 1> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
> ^
> Error:(410, 22) not found: value Records
> val capability =
> Records.newRecord(classOf[Resource]).asInstanceOf[Resource]
> ^
> Error:(410, 72) not found: type Resource
> val capability =
> Records.newRecord(classOf[Resource]).asInstanceOf[Resource]
> ^
> Error:(434, 26) value getApplicationReport is not a member of
> org.apache.spark.Logging
> val report = super.getApplicationReport(appId)
> ^
> Error:(474, 20) not found: type ClientArguments
> val args = new ClientArguments(argStrings, sparkConf)
> ^
> Error:(481, 31) not found: value YarnConfiguration
> for (c <- conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH))
> {
> ^
> Error:(487, 5) not found: value Apps
> Apps.addToEnvironment(env, Environment.CLASSPATH.name,
> Environment.PWD.$())
> ^
> Error:(490, 7) not found: value Apps
> Apps.addToEnvironment(env, Environment.CLASSPATH.name,
> Environment.PWD.$() +
> ^
> Error:(496, 7) not found: value Apps
> Apps.addToEnvironment(env, Environment.CLASSPATH.name,
> Environment.PWD.$() +
> ^
> Error:(499, 5) not found: value Apps
> Apps.addToEnvironment(env, Environment.CLASSPATH.name,
> Environment.PWD.$() +
> ^
> Error:(504, 7) not found: value Apps
> Apps.addToEnvironment(env, Environment.CLASSPATH.name,
> Environment.PWD.$() +
> ^
> Error:(507, 5) not found: value Apps
> Apps.addToEnvironment(env, Environment.CLASSPATH.name,
> Environment.PWD.$() +
> ^
>
> Any idea what's causing them, and maybe I am not using the best practice to
> import Spark to IDE?
>
> I would appreciate any suggestion on the best practice to import Spark to
> any IDE.
>
> Thank you
>
> -Mo
