i'm using
spark2.1+carbon1.1(https://github.com/apache/carbondata/tree/apache-carbondata-1.1.0),and
get class not found exception.and this class only exists in spark1.X import by
CodeGenFactory.scala
build jar using :mvn package -DskipTests -Pspark-2.1 -Dspark.version=2.1.0
-Phadoop-2.7.2
below are error messages(also in file attached):
scala> import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.SparkSession
scala> import org.apache.spark.sql.CarbonSession._
import org.apache.spark.sql.CarbonSession._
scala> val carbon =
SparkSession.builder().config(sc.getConf).getOrCreateCarbonSession("hdfs://192.168.14.78:8020/apps/hive/guoht/qqdatast
17/05/18 10:42:41 WARN SparkContext: Using an existing SparkContext; some
configuration may not take effect.
carbon: org.apache.spark.sql.SparkSession =
org.apache.spark.sql.CarbonSession@2cfd9b0a
scala> carbon.sql("CREATE TABLE IF NOT EXISTS test_table(id string, name
string, city string, age Int) STORED BY 'carbondata'")
java.lang.NoClassDefFoundError: org/apache/spark/sql/catalyst/CatalystConf
at
org.apache.spark.sql.hive.CarbonSessionState.analyzer$lzycompute(CarbonSessionState.scala:127)
at
org.apache.spark.sql.hive.CarbonSessionState.analyzer(CarbonSessionState.scala:126)
at
org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:69)
at
org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:67)
at
org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:50)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:63)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592)
... 50 elided
Caused by: java.lang.ClassNotFoundException:
org.apache.spark.sql.catalyst.CatalystConf
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 57 more
sunerhan1...@sina.com
