[ https://issues.apache.org/jira/browse/HIVE-13930?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Sergey Shelukhin updated HIVE-13930: ------------------------------------ Comment: was deleted (was: Looks like all spark tests fail with this: {noformat} 2016-06-11T15:51:36,748 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - java.lang.NoClassDefFoundError: org/apache/hadoop/crypto/key/KeyProvider 2016-06-11T15:51:36,748 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.lang.Class.forName0(Native Method) 2016-06-11T15:51:36,748 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.lang.Class.forName(Class.java:260) 2016-06-11T15:51:36,748 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.shims.ShimLoader.createShim(ShimLoader.java:143) 2016-06-11T15:51:36,748 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.shims.ShimLoader.loadShims(ShimLoader.java:138) 2016-06-11T15:51:36,748 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.shims.ShimLoader.getHadoopShims(ShimLoader.java:95) 2016-06-11T15:51:36,748 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.io.AcidUtils.<clinit>(AcidUtils.java:115) 2016-06-11T15:51:36,748 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.plan.TableScanDesc.<init>(TableScanDesc.java:129) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.plan.TableScanDesc.<init>(TableScanDesc.java:122) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.plan.TableScanDesc.<init>(TableScanDesc.java:113) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.plan.TableScanDescConstructorAccess.newInstance(Unknown Source) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.Kryo$DefaultInstantiatorStrategy$1.newInstance(Kryo.java:1237) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.Kryo.newInstance(Kryo.java:1090) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.create(FieldSerializer.java:570) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:546) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:708) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.exec.SerializationUtilities$KryoWithHooks.readObject(SerializationUtilities.java:213) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:551) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:790) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.exec.SerializationUtilities$KryoWithHooks.readClassAndObject(SerializationUtilities.java:175) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.read(MapSerializer.java:161) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.read(MapSerializer.java:39) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:708) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.exec.SerializationUtilities$KryoWithHooks.readObject(SerializationUtilities.java:213) 2016-06-11T15:51:36,749 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:551) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:790) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.exec.SerializationUtilities$KryoWithHooks.readClassAndObject(SerializationUtilities.java:175) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.read(MapSerializer.java:153) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.read(MapSerializer.java:39) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:708) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.exec.SerializationUtilities$KryoWithHooks.readObject(SerializationUtilities.java:213) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:551) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:686) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.exec.SerializationUtilities$KryoWithHooks.readObject(SerializationUtilities.java:205) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.exec.spark.KryoSerializer.deserialize(KryoSerializer.java:60) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hadoop.hive.ql.exec.spark.RemoteHiveSparkClient$JobStatusJob.call(RemoteHiveSparkClient.java:317) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.spark.client.RemoteDriver$JobWrapper.call(RemoteDriver.java:358) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at org.apache.hive.spark.client.RemoteDriver$JobWrapper.call(RemoteDriver.java:323) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2016-06-11T15:51:36,750 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.lang.Thread.run(Thread.java:745) 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.crypto.key.KeyProvider 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.net.URLClassLoader$1.run(URLClassLoader.java:372) 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.net.URLClassLoader$1.run(URLClassLoader.java:361) 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.security.AccessController.doPrivileged(Native Method) 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.net.URLClassLoader.findClass(URLClassLoader.java:360) 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.lang.ClassLoader.loadClass(ClassLoader.java:424) 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - at java.lang.ClassLoader.loadClass(ClassLoader.java:357) 2016-06-11T15:51:36,751 INFO [stderr-redir-1[]]: client.SparkClientImpl (SparkClientImpl.java:run(616)) - ... 44 more {noformat} It seems like dependency is there in q file project; the file has not moved since Hadoop 2.6 [~xuefuz] do you have any input on why this would fail? Or is this a known error, and the cause of the failures is something else? The test seems to continue executing for a while after this error. I am looking at other tests. Most seem to be caused by YARN module restructuring.) > upgrade Hive to latest Hadoop version > ------------------------------------- > > Key: HIVE-13930 > URL: https://issues.apache.org/jira/browse/HIVE-13930 > Project: Hive > Issue Type: Bug > Reporter: Sergey Shelukhin > Assignee: Sergey Shelukhin > Attachments: HIVE-13930.01.patch, HIVE-13930.02.patch, > HIVE-13930.patch > > -- This message was sent by Atlassian JIRA (v6.3.4#6332)