Re: java.lang.IncompatibleClassChangeError when using PrunedFilteredScan

2015-03-30 Thread Gaspar Muñoz 
Hello,

Thank you for your contribution.

We have tried to reproduce your error but we need more information:

- Which Spark version are you using? Stratio Spark-Mongodb connector
supports 1.2.x SparkSQL version.

- What jars are you adding while launching the Spark-shell?

Best regards,

2015-03-03 14:06 GMT+01:00 Cheng, Hao hao.ch...@intel.com:

  As the call stack shows, the mongodb connector is not compatible with
 the Spark SQL Data Source interface. The latest Data Source API is changed
 since 1.2, probably you need to confirm which spark version the MongoDB
 Connector build against.



 By the way, a well format call stack will be more helpful for people
 reading.



 *From:* taoewang [mailto:taoew...@sequoiadb.com]
 *Sent:* Tuesday, March 3, 2015 7:39 PM
 *To:* user@spark.apache.org
 *Subject:* java.lang.IncompatibleClassChangeError when using
 PrunedFilteredScan





 Hi,



 I’m trying to build the stratio spark-mongodb connector and got error 
 java.lang.IncompatibleClassChangeError:
 class com.stratio.deep.mongodb.MongodbRelation has interface
 org.apache.spark.sql.sources.PrunedFilteredScan as super class” when trying
 to create a table using the driver:



 scala import org.apache.spark.sql.SQLContext

 val sqlContext = new SQLContext(sc)import org.apache.spark.sql.SQLContext



 import org.apache.spark.sql.SQLContext



 scala val sqlContext = new SQLContext(sc)

 sqlContext: org.apache.spark.sql.SQLContext =
 org.apache.spark.sql.SQLContext@37050c15



 scala import com.stratio.deep.mongodb

 import com.stratio.deep.mongodb

 import com.stratio.deep.mongodb



 scala sqlContext.sql(CREATE TEMPORARY TABLE students_table USING 
 com.stratio.deep.mongodb
 OPTIONS (host 'host:port', database 'highschool', collection 'students'))

 sqlContext.sql(CREATE TEMPORARY TABLE students_table USING com.stratio.d

 eep.mongodb OPTIONS (host 'host:port', database 'highschool', collection
 'studen

 ts'))

 java.lang.IncompatibleClassChangeError: class
 com.stratio.deep.mongodb.MongodbRelation has interface
 org.apache.spark.sql.sources.PrunedFilteredScan as super class

 at java.lang.ClassLoader.defineClass1(Native Method)

 at java.lang.ClassLoader.defineClass(ClassLoader.java:800)

 at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)

 at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)

 at java.net.URLClassLoader.access$100(URLClassLoader.java:71)

 at java.net.URLClassLoader$1.run(URLClassLoader.java:361)

 at java.net.URLClassLoader$1.run(URLClassLoader.java:355)

 at java.security.AccessController.doPrivileged(Native Method)

 at java.net.URLClassLoader.findClass(URLClassLoader.java:354)

 at java.lang.ClassLoader.loadClass(ClassLoader.java:425)

 at java.lang.ClassLoader.loadClass(ClassLoader.java:358)

 at java.lang.Class.getDeclaredConstructors0(Native Method)

 at java.lang.Class.privateGetDeclaredConstructors(Class.java:2585)

 at java.lang.Class.getConstructor0(Class.java:2885)

 at java.lang.Class.newInstance(Class.java:350)

 at org.apache.spark.sql.sources.ResolvedDataSource$.apply(ddl.scala:288)

 at org.apache.spark.sql.sources.CreateTempTableUsing.run(ddl.scala:376)

 at
 org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult$lzycompute(
 commands.scala:55)

 at org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult(
 commands.scala:55)



 The code failed at line 288 in ddl.scala:

   def apply(

   sqlContext: SQLContext,

   userSpecifiedSchema: Option[StructType],

   provider: String,

   options: Map[String, String]): ResolvedDataSource = {

 val clazz: Class[_] = lookupDataSource(provider)

 val relation = userSpecifiedSchema match {

   case Some(schema: StructType) = clazz.newInstance() match {

 case dataSource: SchemaRelationProvider =

   dataSource.createRelation(sqlContext, new CaseInsensitiveMap(
 options), schema)

 case dataSource: org.apache.spark.sql.sources.RelationProvider =

   sys.error(s${clazz.getCanonicalName} does not allow
 user-specified schemas.)

   }



   case None = clazz.newInstance() match {  ——— failed here

 case dataSource: RelationProvider =

   dataSource.createRelation(sqlContext, new CaseInsensitiveMap(
 options))

 case dataSource: org.apache.spark.sql.sources.
 SchemaRelationProvider =

   sys.error(sA schema needs to be specified when using ${clazz.
 getCanonicalName}.)

   }

 }

 new ResolvedDataSource(clazz, relation)

   }



 The “clazz” here is com.stratio.deep.mongodb.DefaultSource, which extends
 RelationProvider:

 class DefaultSource extends RelationProvider {



   override def createRelation(

 sqlContext: SQLContext,

 parameters: Map[String, String]): BaseRelation = {



 /** We will assume hosts are provided like
 'host:port,host2:port2,...'*/

 val host = parameters

   .getOrElse(Host, notFound[String](Host))

   .split(,).toList

RE: java.lang.IncompatibleClassChangeError when using PrunedFilteredScan

2015-03-03 Thread Cheng, Hao 
As the call stack shows, the mongodb connector is not compatible with the Spark 
SQL Data Source interface. The latest Data Source API is changed since 1.2, 
probably you need to confirm which spark version the MongoDB Connector build 
against.

By the way, a well format call stack will be more helpful for people reading.

From: taoewang [mailto:taoew...@sequoiadb.com]
Sent: Tuesday, March 3, 2015 7:39 PM
To: user@spark.apache.org
Subject: java.lang.IncompatibleClassChangeError when using PrunedFilteredScan



Hi,



I’m trying to build the stratio spark-mongodb connector and got error 
java.lang.IncompatibleClassChangeError: class 
com.stratio.deep.mongodb.MongodbRelation has interface 
org.apache.spark.sql.sources.PrunedFilteredScan as super class” when trying to 
create a table using the driver:



scala import org.apache.spark.sql.SQLContext

val sqlContext = new SQLContext(sc)import org.apache.spark.sql.SQLContext



import org.apache.spark.sql.SQLContext



scala val sqlContext = new SQLContext(sc)

sqlContext: org.apache.spark.sql.SQLContext = 
org.apache.spark.sql.SQLContext@37050c15mailto:org.apache.spark.sql.SQLContext@37050c15



scala import com.stratio.deep.mongodb

import com.stratio.deep.mongodb

import com.stratio.deep.mongodb



scala sqlContext.sql(CREATE TEMPORARY TABLE students_table USING 
com.stratio.deep.mongodb OPTIONS (host 'host:port', database 'highschool', 
collection 'students'))

sqlContext.sql(CREATE TEMPORARY TABLE students_table USING com.stratio.d

eep.mongodb OPTIONS (host 'host:port', database 'highschool', collection 'studen

ts'))

java.lang.IncompatibleClassChangeError: class 
com.stratio.deep.mongodb.MongodbRelation has interface 
org.apache.spark.sql.sources.PrunedFilteredScan as super class

at java.lang.ClassLoader.defineClass1(Native Method)

at java.lang.ClassLoader.defineClass(ClassLoader.java:800)

at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)

at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)

at java.net.URLClassLoader.access$100(URLClassLoader.java:71)

at java.net.URLClassLoader$1.run(URLClassLoader.java:361)

at java.net.URLClassLoader$1.run(URLClassLoader.java:355)

at java.security.AccessController.doPrivileged(Native Method)

at java.net.URLClassLoader.findClass(URLClassLoader.java:354)

at java.lang.ClassLoader.loadClass(ClassLoader.java:425)

at java.lang.ClassLoader.loadClass(ClassLoader.java:358)

at java.lang.Class.getDeclaredConstructors0(Native Method)

at java.lang.Class.privateGetDeclaredConstructors(Class.java:2585)

at java.lang.Class.getConstructor0(Class.java:2885)

at java.lang.Class.newInstance(Class.java:350)

at org.apache.spark.sql.sources.ResolvedDataSource$.apply(ddl.scala:288)

at org.apache.spark.sql.sources.CreateTempTableUsing.run(ddl.scala:376)

at 
org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult$lzycompute(commands.scala:55)

at 
org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult(commands.scala:55)



The code failed at line 288 in ddl.scala:

  def apply(

  sqlContext: SQLContext,

  userSpecifiedSchema: Option[StructType],

  provider: String,

  options: Map[String, String]): ResolvedDataSource = {

val clazz: Class[_] = lookupDataSource(provider)

val relation = userSpecifiedSchema match {

  case Some(schema: StructType) = clazz.newInstance() match {

case dataSource: SchemaRelationProvider =

  dataSource.createRelation(sqlContext, new 
CaseInsensitiveMap(options), schema)

case dataSource: org.apache.spark.sql.sources.RelationProvider =

  sys.error(s${clazz.getCanonicalName} does not allow user-specified 
schemas.)

  }



  case None = clazz.newInstance() match {  ——— failed here

case dataSource: RelationProvider =

  dataSource.createRelation(sqlContext, new CaseInsensitiveMap(options))

case dataSource: org.apache.spark.sql.sources.SchemaRelationProvider =

  sys.error(sA schema needs to be specified when using 
${clazz.getCanonicalName}.)

  }

}

new ResolvedDataSource(clazz, relation)

  }



The “clazz” here is com.stratio.deep.mongodb.DefaultSource, which extends 
RelationProvider:

class DefaultSource extends RelationProvider {



  override def createRelation(

sqlContext: SQLContext,

parameters: Map[String, String]): BaseRelation = {



/** We will assume hosts are provided like 'host:port,host2:port2,...'*/

val host = parameters

  .getOrElse(Host, notFound[String](Host))

  .split(,).toList



val database = parameters.getOrElse(Database, notFound(Database))



val collection = parameters.getOrElse(Collection, notFound(Collection))



val samplingRatio = parameters

  .get(SamplingRatio)

  .map(_.toDouble).getOrElse(DefaultSamplingRatio)



MongodbRelation(

  MongodbConfigBuilder()

.set(Host,host)

.set(Database,database)