[ https://issues.apache.org/jira/browse/SPARK-28688?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Yuming Wang updated SPARK-28688: -------------------------------- Summary: Incompatible java.util.ArrayList (was: incompatible java.util.ArrayList) > Incompatible java.util.ArrayList > -------------------------------- > > Key: SPARK-28688 > URL: https://issues.apache.org/jira/browse/SPARK-28688 > Project: Spark > Issue Type: Sub-task > Components: SQL, Tests > Affects Versions: 3.0.0 > Reporter: Yuming Wang > Priority: Major > Attachments: screenshot-1.png, screenshot-2.png > > > {{2 TESTS FAILED}} from {{org.apache.spark.sql.hive.client.VersionsSuite}} > {noformat} > - 3.0: sql read hive materialized view *** FAILED *** (1 second, 292 > milliseconds) > - 3.1: sql read hive materialized view *** FAILED *** (1 second, 312 > milliseconds) > {noformat} > Logs: > {noformat} > 06:54:07.608 WARN org.apache.hadoop.hive.ql.exec.Utilities: Cannot create > class org.apache.hive.storage.jdbc.JdbcInputFormat for > hive.vectorized.row.serde.inputformat.excludes checks > 06:54:07.670 WARN org.apache.hadoop.hive.ql.Driver: Hive-on-MR is deprecated > in Hive 2 and may not be available in the future versions. Consider using a > different execution engine (i.e. spark, tez) or using Hive 1.X releases. > java.lang.RuntimeException: java.lang.NoSuchFieldException: parentOffset > at > org.apache.hadoop.hive.ql.exec.SerializationUtilities$ArrayListSubListSerializer.<init>(SerializationUtilities.java:389) > at > org.apache.hadoop.hive.ql.exec.SerializationUtilities$1.create(SerializationUtilities.java:235) > at > org.apache.hive.com.esotericsoftware.kryo.pool.KryoPoolQueueImpl.borrow(KryoPoolQueueImpl.java:51) > at > org.apache.hadoop.hive.ql.exec.SerializationUtilities.borrowKryo(SerializationUtilities.java:279) > at > org.apache.hadoop.hive.ql.exec.Utilities.setBaseWork(Utilities.java:571) > at > org.apache.hadoop.hive.ql.exec.Utilities.setMapWork(Utilities.java:563) > at > org.apache.hadoop.hive.ql.exec.Utilities.setMapRedWork(Utilities.java:555) > at > org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:362) > at > org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:149) > at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:205) > at > org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:97) > at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2479) > at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:2150) > at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1826) > at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1567) > at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1556) > at > org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$runHive$1(HiveClientImpl.scala:786) > at > org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:310) > at > org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:244) > at > org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:243) > at > org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:293) > at > org.apache.spark.sql.hive.client.HiveClientImpl.runHive(HiveClientImpl.scala:766) > at > org.apache.spark.sql.hive.client.HiveClientImpl.runSqlHive(HiveClientImpl.scala:753) > at > org.apache.spark.sql.hive.client.VersionsSuite.$anonfun$new$72(VersionsSuite.scala:626) > at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85) > at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83) > at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) > at org.scalatest.Transformer.apply(Transformer.scala:22) > at org.scalatest.Transformer.apply(Transformer.scala:20) > at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186) > at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:149) > at > org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184) > at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196) > at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289) > at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196) > at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178) > at > org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:56) > at > org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221) > at > org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214) > at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:56) > at > org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229) > at > org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:396) > at scala.collection.immutable.List.foreach(List.scala:392) > at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384) > at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:379) > at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461) > at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229) > at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228) > at org.scalatest.FunSuite.runTests(FunSuite.scala:1560) > at org.scalatest.Suite.run(Suite.scala:1147) > at org.scalatest.Suite.run$(Suite.scala:1129) > at > org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560) > at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233) > at org.scalatest.SuperEngine.runImpl(Engine.scala:521) > at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233) > at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232) > at > org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:56) > at > org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213) > at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210) > at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) > at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:56) > at > org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:314) > at > org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:507) > at sbt.ForkMain$Run$2.call(ForkMain.java:296) > at sbt.ForkMain$Run$2.call(ForkMain.java:286) > at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > at java.base/java.lang.Thread.run(Thread.java:834) > Caused by: java.lang.NoSuchFieldException: parentOffset > at java.base/java.lang.Class.getDeclaredField(Class.java:2412) > at > org.apache.hadoop.hive.ql.exec.SerializationUtilities$ArrayListSubListSerializer.<init>(SerializationUtilities.java:383) > ... 68 more > 06:54:07.780 ERROR org.apache.hadoop.hive.ql.exec.Task: Job Submission failed > with exception 'java.lang.RuntimeException(java.lang.NoSuchFieldException: > parentOffset)' > java.lang.RuntimeException: java.lang.NoSuchFieldException: parentOffset > at > org.apache.hadoop.hive.ql.exec.SerializationUtilities$ArrayListSubListSerializer.<init>(SerializationUtilities.java:389) > at > org.apache.hadoop.hive.ql.exec.SerializationUtilities$1.create(SerializationUtilities.java:235) > at > org.apache.hive.com.esotericsoftware.kryo.pool.KryoPoolQueueImpl.borrow(KryoPoolQueueImpl.java:51) > at > org.apache.hadoop.hive.ql.exec.SerializationUtilities.borrowKryo(SerializationUtilities.java:279) > at > org.apache.hadoop.hive.ql.exec.Utilities.setBaseWork(Utilities.java:571) > at > org.apache.hadoop.hive.ql.exec.Utilities.setMapWork(Utilities.java:563) > at > org.apache.hadoop.hive.ql.exec.Utilities.setMapRedWork(Utilities.java:555) > at > org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:362) > at > org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:149) > at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:205) > at > org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:97) > at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2479) > at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:2150) > at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1826) > at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1567) > at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1556) > at > org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$runHive$1(HiveClientImpl.scala:786) > at > org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:310) > at > org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:244) > at > org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:243) > at > org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:293) > at > org.apache.spark.sql.hive.client.HiveClientImpl.runHive(HiveClientImpl.scala:766) > at > org.apache.spark.sql.hive.client.HiveClientImpl.runSqlHive(HiveClientImpl.scala:753) > at > org.apache.spark.sql.hive.client.VersionsSuite.$anonfun$new$72(VersionsSuite.scala:626) > at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85) > at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83) > at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) > at org.scalatest.Transformer.apply(Transformer.scala:22) > at org.scalatest.Transformer.apply(Transformer.scala:20) > at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186) > at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:149) > at > org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184) > at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196) > at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289) > at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196) > at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178) > at > org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:56) > at > org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221) > at > org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214) > at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:56) > at > org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229) > at > org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:396) > at scala.collection.immutable.List.foreach(List.scala:392) > at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384) > at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:379) > at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461) > at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229) > at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228) > at org.scalatest.FunSuite.runTests(FunSuite.scala:1560) > at org.scalatest.Suite.run(Suite.scala:1147) > at org.scalatest.Suite.run$(Suite.scala:1129) > at > org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560) > at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233) > at org.scalatest.SuperEngine.runImpl(Engine.scala:521) > at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233) > at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232) > at > org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:56) > at > org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213) > at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210) > at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) > at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:56) > at > org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:314) > at > org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:507) > at sbt.ForkMain$Run$2.call(ForkMain.java:296) > at sbt.ForkMain$Run$2.call(ForkMain.java:286) > at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > at java.base/java.lang.Thread.run(Thread.java:834) > Caused by: java.lang.NoSuchFieldException: parentOffset > at java.base/java.lang.Class.getDeclaredField(Class.java:2412) > at > org.apache.hadoop.hive.ql.exec.SerializationUtilities$ArrayListSubListSerializer.<init>(SerializationUtilities.java:383) > ... 68 more > 06:54:07.780 ERROR org.apache.hadoop.hive.ql.Driver: FAILED: Execution Error, > return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask. > java.lang.NoSuchFieldException: parentOffset > 06:54:07.792 ERROR org.apache.spark.sql.hive.client.HiveClientImpl: > ====================== > HIVE FAILURE OUTPUT > ====================== > SET spark.sql.test.key=1 > OK > Query ID = root_20190811065407_baceadf3-30bb-4884-a815-72cbb02d429d > Total jobs = 1 > Launching Job 1 out of 1 > Number of reduce tasks is set to 0 since there's no reduce operator > Job Submission failed with exception > 'java.lang.RuntimeException(java.lang.NoSuchFieldException: parentOffset)' > FAILED: Execution Error, return code 1 from > org.apache.hadoop.hive.ql.exec.mr.MapRedTask. java.lang.NoSuchFieldException: > parentOffset > ====================== > END HIVE FAILURE OUTPUT > ====================== > [info] - 3.0: sql read hive materialized view *** FAILED *** (1 second, 292 > milliseconds) > [info] org.apache.spark.sql.execution.QueryExecutionException: FAILED: > Execution Error, return code 1 from > org.apache.hadoop.hive.ql.exec.mr.MapRedTask. java.lang.NoSuchFieldException: > parentOffset > [info] at > org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$runHive$1(HiveClientImpl.scala:790) > [info] at > org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:310) > [info] at > org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:244) > [info] at > org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:243) > [info] at > org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:293) > [info] at > org.apache.spark.sql.hive.client.HiveClientImpl.runHive(HiveClientImpl.scala:766) > [info] at > org.apache.spark.sql.hive.client.HiveClientImpl.runSqlHive(HiveClientImpl.scala:753) > [info] at > org.apache.spark.sql.hive.client.VersionsSuite.$anonfun$new$72(VersionsSuite.scala:626) > [info] at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85) > [info] at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83) > [info] at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) > [info] at org.scalatest.Transformer.apply(Transformer.scala:22) > [info] at org.scalatest.Transformer.apply(Transformer.scala:20) > [info] at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186) > [info] at > org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:149) > [info] at > org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184) > [info] at > org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196) > [info] at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289) > [info] at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196) > [info] at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178) > [info] at > org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:56) > [info] at > org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221) > [info] at > org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214) > [info] at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:56) > [info] at > org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229) > [info] at > org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:396) > [info] at scala.collection.immutable.List.foreach(List.scala:392) > [info] at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384) > [info] at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:379) > [info] at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461) > [info] at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229) > [info] at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228) > [info] at org.scalatest.FunSuite.runTests(FunSuite.scala:1560) > [info] at org.scalatest.Suite.run(Suite.scala:1147) > [info] at org.scalatest.Suite.run$(Suite.scala:1129) > [info] at > org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560) > [info] at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233) > [info] at org.scalatest.SuperEngine.runImpl(Engine.scala:521) > [info] at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233) > [info] at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232) > [info] at > org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:56) > [info] at > org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213) > [info] at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210) > [info] at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) > [info] at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:56) > [info] at > org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:314) > [info] at > org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:507) > [info] at sbt.ForkMain$Run$2.call(ForkMain.java:296) > [info] at sbt.ForkMain$Run$2.call(ForkMain.java:286) > [info] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) > [info] at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > [info] at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > [info] at java.base/java.lang.Thread.run(Thread.java:834) > {noformat} > https://github.com/apache/hive/blob/ae4df627952610dbec029b099f0964908b3a4f25/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java#L381-L383 > !screenshot-2.png! > !screenshot-1.png! -- This message was sent by Atlassian JIRA (v7.6.14#76016) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org