You need to have the datanuclus jars on your classpath. It is not okay to merge them into an uber jar.
On Wed, Aug 27, 2014 at 1:44 AM, centerqi hu <cente...@gmail.com> wrote: > Hi all > > > When I run a simple SQL, encountered the following error. > > hive:0.12(metastore in mysql) > > hadoop 2.4.1 > > spark 1.0.2 build with hive > > > my hql code > > import org.apache.spark.{SparkConf, SparkContext} > import org.apache.spark.sql._ > import org.apache.spark.sql.hive.LocalHiveContext > object HqlTest { > case class Record(key: Int, value: String) > def main(args: Array[String]) { > val sparkConf = new SparkConf().setAppName("HiveFromSpark") > val sc = new SparkContext(sparkConf) > val hiveContext = new org.apache.spark.sql.hive.HiveContext(sc) > import hiveContext._ > hql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)") > } > > } > > > > > > > 14/08/27 16:07:08 INFO HiveMetaStore: 0: Opening raw store with > implemenation class:org.apache.hadoop.hive.metastore.ObjectStore > 14/08/27 16:07:08 INFO ObjectStore: ObjectStore, initialize called > org.apache.hadoop.hive.ql.metadata.HiveException: Unable to fetch table src > at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:958) > at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:905) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeCreateTable(SemanticAnalyzer.java:8999) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:8313) > at > org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:284) > at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:441) > at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:342) > at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:977) > at org.apache.hadoop.hive.ql.Driver.run(Driver.java:888) > at org.apache.spark.sql.hive.HiveContext.runHive(HiveContext.scala:189) > at org.apache.spark.sql.hive.HiveContext.runSqlHive(HiveContext.scala:163) > at > org.apache.spark.sql.hive.execution.NativeCommand.sideEffectResult$lzycompute(NativeCommand.scala:35) > at > org.apache.spark.sql.hive.execution.NativeCommand.sideEffectResult(NativeCommand.scala:35) > at > org.apache.spark.sql.hive.execution.NativeCommand.execute(NativeCommand.scala:38) > at > org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd$lzycompute(HiveContext.scala:250) > at > org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd(HiveContext.scala:250) > at org.apache.spark.sql.SchemaRDDLike$class.$init$(SchemaRDDLike.scala:58) > at org.apache.spark.sql.SchemaRDD.<init>(SchemaRDD.scala:104) > at org.apache.spark.sql.hive.HiveContext.hiveql(HiveContext.scala:75) > at org.apache.spark.sql.hive.HiveContext.hql(HiveContext.scala:78) > at HqlTest$.main(HqlTest.scala:15) > at HqlTest.main(HqlTest.scala) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:186) > Caused by: java.lang.RuntimeException: Unable to instantiate > org.apache.hadoop.hive.metastore.HiveMetaStoreClient > at > org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1212) > at > org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:62) > at > org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:72) > at > org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:2372) > at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:2383) > at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:950) > ... 26 more > Caused by: java.lang.reflect.InvocationTargetException > at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) > at > sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) > at > sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) > at java.lang.reflect.Constructor.newInstance(Constructor.java:526) > at > org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1210) > ... 31 more > Caused by: javax.jdo.JDOFatalUserException: Class > org.datanucleus.api.jdo.JDOPersistenceManagerFactory was not found. > NestedThrowables: > java.lang.ClassNotFoundException: > org.datanucleus.api.jdo.JDOPersistenceManagerFactory > at > javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1175) > at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) > at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) > at > org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:275) > at > org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:304) > at > org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:234) > at > org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:209) > at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > at > org.apache.hadoop.hive.metastore.RetryingRawStore.<init>(RetryingRawStore.java:64) > at > org.apache.hadoop.hive.metastore.RetryingRawStore.getProxy(RetryingRawStore.java:73) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:415) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:402) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:441) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:326) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:286) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:54) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:59) > at > org.apache.hadoop.hive.metastore.HiveMetaStore.newHMSHandler(HiveMetaStore.java:4060) > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:121) > ... 36 more > Caused by: java.lang.ClassNotFoundException: > org.datanucleus.api.jdo.JDOPersistenceManagerFactory > at java.net.URLClassLoader$1.run(URLClassLoader.java:366) > at java.net.URLClassLoader$1.run(URLClassLoader.java:355) > at java.security.AccessController.doPrivileged(Native Method) > at java.net.URLClassLoader.findClass(URLClassLoader.java:354) > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > at java.lang.Class.forName0(Native Method) > at java.lang.Class.forName(Class.java:270) > at javax.jdo.JDOHelper$18.run(JDOHelper.java:2018) > at javax.jdo.JDOHelper$18.run(JDOHelper.java:2016) > at java.security.AccessController.doPrivileged(Native Method) > at javax.jdo.JDOHelper.forName(JDOHelper.java:2015) > at > javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1162) > ... 55 more > 14/08/27 16:07:08 INFO Driver: Semantic Analysis Completed > 14/08/27 16:07:08 INFO Driver: </PERFLOG method=semanticAnalyze > start=1409126828417 end=1409126828624 duration=207> > 14/08/27 16:07:08 INFO Driver: Returning Hive schema: > Schema(fieldSchemas:null, properties:null) > 14/08/27 16:07:08 INFO Driver: </PERFLOG method=compile > start=1409126828395 end=1409126828631 duration=236> > 14/08/27 16:07:08 INFO Driver: <PERFLOG method=Driver.execute> > 14/08/27 16:07:08 INFO deprecation: mapred.job.name is deprecated. > Instead, use mapreduce.job.name > 14/08/27 16:07:08 INFO Driver: Starting command: CREATE TABLE IF NOT > EXISTS src (key INT, value STRING) > 14/08/27 16:07:08 INFO Driver: </PERFLOG method=TimeToSubmit > start=1409126828394 end=1409126828634 duration=240> > 14/08/27 16:07:08 INFO Driver: <PERFLOG method=runTasks> > 14/08/27 16:07:08 INFO Driver: <PERFLOG method=task.DDL.Stage-0> > 14/08/27 16:07:08 INFO DDLTask: Default to LazySimpleSerDe for table src > 14/08/27 16:07:08 INFO HiveMetaStore: 0: Opening raw store with > implemenation class:org.apache.hadoop.hive.metastore.ObjectStore > 14/08/27 16:07:08 INFO ObjectStore: ObjectStore, initialize called > 14/08/27 16:07:08 ERROR DDLTask: > org.apache.hadoop.hive.ql.metadata.HiveException: > java.lang.RuntimeException: Unable to instantiate > org.apache.hadoop.hive.metastore.HiveMetaStoreClient > at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:602) > at org.apache.hadoop.hive.ql.exec.DDLTask.createTable(DDLTask.java:3661) > at org.apache.hadoop.hive.ql.exec.DDLTask.execute(DDLTask.java:252) > at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:151) > at > org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:65) > at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1414) > at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1192) > at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1020) > at org.apache.hadoop.hive.ql.Driver.run(Driver.java:888) > at org.apache.spark.sql.hive.HiveContext.runHive(HiveContext.scala:189) > at org.apache.spark.sql.hive.HiveContext.runSqlHive(HiveContext.scala:163) > at > org.apache.spark.sql.hive.execution.NativeCommand.sideEffectResult$lzycompute(NativeCommand.scala:35) > at > org.apache.spark.sql.hive.execution.NativeCommand.sideEffectResult(NativeCommand.scala:35) > at > org.apache.spark.sql.hive.execution.NativeCommand.execute(NativeCommand.scala:38) > at > org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd$lzycompute(HiveContext.scala:250) > at > org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd(HiveContext.scala:250) > at org.apache.spark.sql.SchemaRDDLike$class.$init$(SchemaRDDLike.scala:58) > at org.apache.spark.sql.SchemaRDD.<init>(SchemaRDD.scala:104) > at org.apache.spark.sql.hive.HiveContext.hiveql(HiveContext.scala:75) > at org.apache.spark.sql.hive.HiveContext.hql(HiveContext.scala:78) > at HqlTest$.main(HqlTest.scala:15) > at HqlTest.main(HqlTest.scala) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:186) > Caused by: java.lang.RuntimeException: Unable to instantiate > org.apache.hadoop.hive.metastore.HiveMetaStoreClient > at > org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1212) > at > org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:62) > at > org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:72) > at > org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:2372) > at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:2383) > at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:596) > ... 26 more > Caused by: java.lang.reflect.InvocationTargetException > at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) > at > sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) > at > sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) > at java.lang.reflect.Constructor.newInstance(Constructor.java:526) > at > org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1210) > ... 31 more > Caused by: javax.jdo.JDOFatalUserException: Class > org.datanucleus.api.jdo.JDOPersistenceManagerFactory was not found. > NestedThrowables: > java.lang.ClassNotFoundException: > org.datanucleus.api.jdo.JDOPersistenceManagerFactory > at > javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1175) > at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) > at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) > at > org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:275) > at > org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:304) > at > org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:234) > at > org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:209) > at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) > at > org.apache.hadoop.hive.metastore.RetryingRawStore.<init>(RetryingRawStore.java:64) > at > org.apache.hadoop.hive.metastore.RetryingRawStore.getProxy(RetryingRawStore.java:73) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:415) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:402) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:441) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:326) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:286) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:54) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:59) > at > org.apache.hadoop.hive.metastore.HiveMetaStore.newHMSHandler(HiveMetaStore.java:4060) > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:121) > ... 36 more > Caused by: java.lang.ClassNotFoundException: > org.datanucleus.api.jdo.JDOPersistenceManagerFactory > at java.net.URLClassLoader$1.run(URLClassLoader.java:366) > at java.net.URLClassLoader$1.run(URLClassLoader.java:355) > at java.security.AccessController.doPrivileged(Native Method) > at java.net.URLClassLoader.findClass(URLClassLoader.java:354) > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > at java.lang.Class.forName0(Native Method) > at java.lang.Class.forName(Class.java:270) > at javax.jdo.JDOHelper$18.run(JDOHelper.java:2018) > at javax.jdo.JDOHelper$18.run(JDOHelper.java:2016) > at java.security.AccessController.doPrivileged(Native Method) > at javax.jdo.JDOHelper.forName(JDOHelper.java:2015) > at > javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1162) > ... 55 more > > 14/08/27 16:07:08 INFO Driver: </PERFLOG method=task.DDL.Stage-0 > start=1409126828634 end=1409126828682 duration=48> > 14/08/27 16:07:08 ERROR Driver: FAILED: Execution Error, return code 1 > from org.apache.hadoop.hive.ql.exec.DDLTask. > java.lang.RuntimeException: Unable to instantiate > org.apache.hadoop.hive.metastore.HiveMetaStoreClient > 14/08/27 16:07:08 INFO Driver: </PERFLOG method=Driver.execute > start=1409126828631 end=1409126828686 duration=55> > 14/08/27 16:07:08 INFO Driver: <PERFLOG method=releaseLocks> > 14/08/27 16:07:08 INFO Driver: </PERFLOG method=releaseLocks > start=1409126828686 end=1409126828686 duration=0> > 14/08/27 16:07:08 INFO Driver: <PERFLOG method=releaseLocks> > 14/08/27 16:07:08 INFO Driver: </PERFLOG method=releaseLocks > start=1409126828687 end=1409126828687 duration=0> > 14/08/27 16:07:08 ERROR HiveContext: > ====================== > HIVE FAILURE OUTPUT > ====================== > FAILED: Execution Error, return code 1 from > org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.RuntimeException: > Unable to instantiate > org.apache.hadoop.hive.metastore.HiveMetaStoreClient > > ====================== > END HIVE FAILURE OUTPUT > ====================== > > 14/08/27 16:07:08 INFO ApplicationMaster: finishApplicationMaster with > FAILED > 14/08/27 16:07:08 INFO AMRMClientImpl: Waiting for application to be > successfully unregistered. > Exception in thread "Thread-4" java.lang.reflect.InvocationTargetException > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:186) > Caused by: org.apache.spark.sql.execution.QueryExecutionException: > FAILED: Execution Error, return code 1 from > org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.RuntimeException: > Unable to instantiate > org.apache.hadoop.hive.metastore.HiveMetaStoreClient > at org.apache.spark.sql.hive.HiveContext.runHive(HiveContext.scala:193) > at org.apache.spark.sql.hive.HiveContext.runSqlHive(HiveContext.scala:163) > at > org.apache.spark.sql.hive.execution.NativeCommand.sideEffectResult$lzycompute(NativeCommand.scala:35) > at > org.apache.spark.sql.hive.execution.NativeCommand.sideEffectResult(NativeCommand.scala:35) > at > org.apache.spark.sql.hive.execution.NativeCommand.execute(NativeCommand.scala:38) > at > org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd$lzycompute(HiveContext.scala:250) > at > org.apache.spark.sql.hive.HiveContext$QueryExecution.toRdd(HiveContext.scala:250) > at org.apache.spark.sql.SchemaRDDLike$class.$init$(SchemaRDDLike.scala:58) > at org.apache.spark.sql.SchemaRDD.<init>(SchemaRDD.scala:104) > at org.apache.spark.sql.hive.HiveContext.hiveql(HiveContext.scala:75) > at org.apache.spark.sql.hive.HiveContext.hql(HiveContext.scala:78) > at HqlTest$.main(HqlTest.scala:15) > at HqlTest.main(HqlTest.scala) > ... 5 more > > > -- > cente...@gmail.com|齐忠 > > --------------------------------------------------------------------- > To unsubscribe, e-mail: user-unsubscr...@spark.apache.org > For additional commands, e-mail: user-h...@spark.apache.org > >