[ https://issues.apache.org/jira/browse/SPARK-27913?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17087813#comment-17087813 ]
Giri Dandu edited comment on SPARK-27913 at 4/20/20, 2:52 PM: -------------------------------------------------------------- [~hyukjin.kwon] with mergeSchema orc option it is still *NOT* working in spark 2.4.5 {code:java} scala> spark.conf.getAllscala> spark.conf.getAllres15: Map[String,String] = Map(spark.driver.host -> 192.168.7.124, spark.sql.orc.mergeSchema -> true, spark.driver.port -> 54231, spark.repl.class.uri -> spark://192.168.7.124:54231/classes, spark.jars -> "", spark.repl.class.outputDir -> /private/var/folders/6h/nkpdlpcd0h34sq6x2fmz896wt2p_sp/T/spark-373735c9-6837-4734-bb13-e8457848a70e/repl-0852551f-cfa5-4b4a-aa2c-ac129818bbc2, spark.app.name -> Spark shell, spark.ui.showConsoleProgress -> true, spark.executor.id -> driver, spark.submit.deployMode -> client, spark.master -> local[*], spark.home -> /Users/gdandu/Downloads/spark-2.4.5-bin-hadoop2.7, spark.sql.catalogImplementation -> hive, spark.app.id -> local-1587393426045) scala> spark.sql("drop table test_broken_orc"); res16: org.apache.spark.sql.DataFrame = [] scala> spark.sql("create external table test_broken_orc(a struct<f1:int>) stored as orc location '/tmp/test_broken_2'"); res17: org.apache.spark.sql.DataFrame = [] scala> spark.sql("insert into table test_broken_orc select named_struct(\"f1\", 1)"); res18: org.apache.spark.sql.DataFrame = [] scala> spark.sql("select * from test_broken_orc"); res19: org.apache.spark.sql.DataFrame = [a: struct<f1: int>] scala> res19.show +---+| a|+---+|[1]|+---+ scala> spark.sql("drop table test_broken_orc"); res21: org.apache.spark.sql.DataFrame = [] scala> spark.sql("create external table test_broken_orc(a struct<f1:int, f2: int>) stored as orc location '/tmp/test_broken_2'"); res22: org.apache.spark.sql.DataFrame = [] scala> spark.sql("select * from test_broken_orc"); res23: org.apache.spark.sql.DataFrame = [a: struct<f1: int, f2: int>] scala> res23.show 20/04/20 10:46:23 ERROR Executor: Exception in task 0.0 in stage 5.0 (TID 5)java.lang.ArrayIndexOutOfBoundsException: 1 at org.apache.orc.mapred.OrcStruct.getFieldValue(OrcStruct.java:49) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer$$anonfun$org$apache$spark$sql$execution$datasources$orc$OrcDeserializer$$newWriter$14.apply(OrcDeserializer.scala:133) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer$$anonfun$org$apache$spark$sql$execution$datasources$orc$OrcDeserializer$$newWriter$14.apply(OrcDeserializer.scala:123) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer$$anonfun$2$$anonfun$apply$1.apply(OrcDeserializer.scala:51) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer$$anonfun$2$$anonfun$apply$1.apply(OrcDeserializer.scala:51) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer.deserialize(OrcDeserializer.scala:64) at org.apache.spark.sql.execution.datasources.orc.OrcFileFormat$$anonfun$buildReaderWithPartitionValues$2$$anonfun$apply$7.apply(OrcFileFormat.scala:230) at org.apache.spark.sql.execution.datasources.orc.OrcFileFormat$$anonfun$buildReaderWithPartitionValues$2$$anonfun$apply$7.apply(OrcFileFormat.scala:230) at scala.collection.Iterator$$anon$11.next(Iterator.scala:410) at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.next(FileScanRDD.scala:104) at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$13$$anon$1.hasNext(WholeStageCodegenExec.scala:636) at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:255) at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:247) at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:858) at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:858) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:346) at org.apache.spark.rdd.RDD.iterator(RDD.scala:310) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:346) at org.apache.spark.rdd.RDD.iterator(RDD.scala:310) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:123) at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)20/04/20 10:46:23 WARN TaskSetManager: Lost task 0.0 in stage 5.0 (TID 5, localhost, executor driver): java.lang.ArrayIndexOutOfBoundsException: 1 at org.apache.orc.mapred.OrcStruct.getFieldValue(OrcStruct.java:49) {code} was (Author: giri.dandu): [~hyukjin.kwon] with mergeSchema orc option it is still *NOT* working in spark 2.4.5 {code:java} scala> spark.conf.getAllscala> spark.conf.getAllres15: Map[String,String] = Map(spark.driver.host -> 192.168.7.124, spark.sql.orc.mergeSchema -> true, spark.driver.port -> 54231, spark.repl.class.uri -> spark://192.168.7.124:54231/classes, spark.jars -> "", spark.repl.class.outputDir -> /private/var/folders/6h/nkpdlpcd0h34sq6x2fmz896wt2p_sp/T/spark-373735c9-6837-4734-bb13-e8457848a70e/repl-0852551f-cfa5-4b4a-aa2c-ac129818bbc2, spark.app.name -> Spark shell, spark.ui.showConsoleProgress -> true, spark.executor.id -> driver, spark.submit.deployMode -> client, spark.master -> local[*], spark.home -> /Users/gdandu/Downloads/spark-2.4.5-bin-hadoop2.7, spark.sql.catalogImplementation -> hive, spark.app.id -> local-1587393426045) scala> spark.sql("drop table test_broken_orc"); res16: org.apache.spark.sql.DataFrame = [] scala> spark.sql("create external table test_broken_orc(a struct<f1:int>) stored as orc location '/tmp/test_broken_2'");res17: org.apache.spark.sql.DataFrame = [] scala> spark.sql("insert into table test_broken_orc select named_struct(\"f1\", 1)"); res18: org.apache.spark.sql.DataFrame = [] scala> spark.sql("select * from test_broken_orc"); res19: org.apache.spark.sql.DataFrame = [a: struct<f1: int>] scala> res19.show +---+| a|+---+|[1]|+---+ scala> spark.sql("drop table test_broken_orc"); res21: org.apache.spark.sql.DataFrame = [] scala> spark.sql("create external table test_broken_orc(a struct<f1:int, f2: int>) stored as orc location '/tmp/test_broken_2'"); res22: org.apache.spark.sql.DataFrame = [] scala> spark.sql("select * from test_broken_orc"); res23: org.apache.spark.sql.DataFrame = [a: struct<f1: int, f2: int>] scala> res23.show 20/04/20 10:46:23 ERROR Executor: Exception in task 0.0 in stage 5.0 (TID 5)java.lang.ArrayIndexOutOfBoundsException: 1 at org.apache.orc.mapred.OrcStruct.getFieldValue(OrcStruct.java:49) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer$$anonfun$org$apache$spark$sql$execution$datasources$orc$OrcDeserializer$$newWriter$14.apply(OrcDeserializer.scala:133) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer$$anonfun$org$apache$spark$sql$execution$datasources$orc$OrcDeserializer$$newWriter$14.apply(OrcDeserializer.scala:123) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer$$anonfun$2$$anonfun$apply$1.apply(OrcDeserializer.scala:51) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer$$anonfun$2$$anonfun$apply$1.apply(OrcDeserializer.scala:51) at org.apache.spark.sql.execution.datasources.orc.OrcDeserializer.deserialize(OrcDeserializer.scala:64) at org.apache.spark.sql.execution.datasources.orc.OrcFileFormat$$anonfun$buildReaderWithPartitionValues$2$$anonfun$apply$7.apply(OrcFileFormat.scala:230) at org.apache.spark.sql.execution.datasources.orc.OrcFileFormat$$anonfun$buildReaderWithPartitionValues$2$$anonfun$apply$7.apply(OrcFileFormat.scala:230) at scala.collection.Iterator$$anon$11.next(Iterator.scala:410) at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.next(FileScanRDD.scala:104) at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$13$$anon$1.hasNext(WholeStageCodegenExec.scala:636) at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:255) at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:247) at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:858) at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:858) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:346) at org.apache.spark.rdd.RDD.iterator(RDD.scala:310) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:346) at org.apache.spark.rdd.RDD.iterator(RDD.scala:310) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:123) at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)20/04/20 10:46:23 WARN TaskSetManager: Lost task 0.0 in stage 5.0 (TID 5, localhost, executor driver): java.lang.ArrayIndexOutOfBoundsException: 1 at org.apache.orc.mapred.OrcStruct.getFieldValue(OrcStruct.java:49) {code} > Spark SQL's native ORC reader implements its own schema evolution > ----------------------------------------------------------------- > > Key: SPARK-27913 > URL: https://issues.apache.org/jira/browse/SPARK-27913 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 2.3.3 > Reporter: Owen O'Malley > Priority: Major > > ORC's reader handles a wide range of schema evolution, but the Spark SQL > native ORC bindings do not provide the desired schema to the ORC reader. This > causes a regression when moving spark.sql.orc.impl from 'hive' to 'native'. -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org