[ https://issues.apache.org/jira/browse/SPARK-24204?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Dongjoon Hyun updated SPARK-24204: ---------------------------------- Description: *SUMMARY* - CSV: Raising analysis exception. - JSON: dropping columns with null types - Parquet/ORC: raising runtime exceptions The native orc file format throws an exception with a meaningless message in executor-sides when unsupported types passed; {code} scala> val rdd = spark.sparkContext.parallelize(List(Row(1, null), Row(2, null))) scala> val schema = StructType(StructField("a", IntegerType) :: StructField("b", NullType) :: Nil) scala> val df = spark.createDataFrame(rdd, schema) scala> df.write.orc("/tmp/orc") java.lang.IllegalArgumentException: Can't parse category at 'struct<a:int,b:null^>' at org.apache.orc.TypeDescription.parseCategory(TypeDescription.java:223) at org.apache.orc.TypeDescription.parseType(TypeDescription.java:332) at org.apache.orc.TypeDescription.parseStruct(TypeDescription.java:327) at org.apache.orc.TypeDescription.parseType(TypeDescription.java:385) at org.apache.orc.TypeDescription.fromString(TypeDescription.java:406) at org.apache.spark.sql.execution.datasources.orc.OrcSerializer.org$apache$spark$sql$execution$datasources$orc$OrcSerializer$$createOrcValue(OrcSerializ er.scala:226) at org.apache.spark.sql.execution.datasources.orc.OrcSerializer.<init>(OrcSerializer.scala:36) at org.apache.spark.sql.execution.datasources.orc.OrcOutputWriter.<init>(OrcOutputWriter.scala:36) at org.apache.spark.sql.execution.datasources.orc.OrcFileFormat$$anon$1.newInstance(OrcFileFormat.scala:108) at org.apache.spark.sql.execution.datasources.FileFormatWriter$SingleDirectoryWriteTask.newOutputWriter(FileFormatWriter.scala:376) at org.apache.spark.sql.execution.datasources.FileFormatWriter$SingleDirectoryWriteTask.execute(FileFormatWriter.scala:387) at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask$3.apply (FileFormatWriter.scala:278) {code} It seems to be better to verify a write schema in a driver side for users along with the CSV fromat; https://github.com/apache/spark/blob/76ecd095024a658bf68e5db658e4416565b30c17/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVFileFormat.scala#L65 was: The native orc file format throws an exception with a meaningless message in executor-sides when unsupported types passed; {code} scala> val rdd = spark.sparkContext.parallelize(List(Row(1, null), Row(2, null))) scala> val schema = StructType(StructField("a", IntegerType) :: StructField("b", NullType) :: Nil) scala> val df = spark.createDataFrame(rdd, schema) scala> df.write.orc("/tmp/orc") java.lang.IllegalArgumentException: Can't parse category at 'struct<a:int,b:null^>' at org.apache.orc.TypeDescription.parseCategory(TypeDescription.java:223) at org.apache.orc.TypeDescription.parseType(TypeDescription.java:332) at org.apache.orc.TypeDescription.parseStruct(TypeDescription.java:327) at org.apache.orc.TypeDescription.parseType(TypeDescription.java:385) at org.apache.orc.TypeDescription.fromString(TypeDescription.java:406) at org.apache.spark.sql.execution.datasources.orc.OrcSerializer.org$apache$spark$sql$execution$datasources$orc$OrcSerializer$$createOrcValue(OrcSerializ er.scala:226) at org.apache.spark.sql.execution.datasources.orc.OrcSerializer.<init>(OrcSerializer.scala:36) at org.apache.spark.sql.execution.datasources.orc.OrcOutputWriter.<init>(OrcOutputWriter.scala:36) at org.apache.spark.sql.execution.datasources.orc.OrcFileFormat$$anon$1.newInstance(OrcFileFormat.scala:108) at org.apache.spark.sql.execution.datasources.FileFormatWriter$SingleDirectoryWriteTask.newOutputWriter(FileFormatWriter.scala:376) at org.apache.spark.sql.execution.datasources.FileFormatWriter$SingleDirectoryWriteTask.execute(FileFormatWriter.scala:387) at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask$3.apply (FileFormatWriter.scala:278) {code} It seems to be better to verify a write schema in a driver side for users along with the CSV fromat; https://github.com/apache/spark/blob/76ecd095024a658bf68e5db658e4416565b30c17/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVFileFormat.scala#L65 > Verify a write schema in Json/Orc/ParquetFileFormat > --------------------------------------------------- > > Key: SPARK-24204 > URL: https://issues.apache.org/jira/browse/SPARK-24204 > Project: Spark > Issue Type: Improvement > Components: SQL > Affects Versions: 2.3.0 > Reporter: Takeshi Yamamuro > Priority: Minor > > *SUMMARY* > - CSV: Raising analysis exception. > - JSON: dropping columns with null types > - Parquet/ORC: raising runtime exceptions > The native orc file format throws an exception with a meaningless message in > executor-sides when unsupported types passed; > {code} > scala> val rdd = spark.sparkContext.parallelize(List(Row(1, null), Row(2, > null))) > scala> val schema = StructType(StructField("a", IntegerType) :: > StructField("b", NullType) :: Nil) > scala> val df = spark.createDataFrame(rdd, schema) > scala> df.write.orc("/tmp/orc") > java.lang.IllegalArgumentException: Can't parse category at > 'struct<a:int,b:null^>' > at > org.apache.orc.TypeDescription.parseCategory(TypeDescription.java:223) > at org.apache.orc.TypeDescription.parseType(TypeDescription.java:332) > at > org.apache.orc.TypeDescription.parseStruct(TypeDescription.java:327) > at org.apache.orc.TypeDescription.parseType(TypeDescription.java:385) > at org.apache.orc.TypeDescription.fromString(TypeDescription.java:406) > at > org.apache.spark.sql.execution.datasources.orc.OrcSerializer.org$apache$spark$sql$execution$datasources$orc$OrcSerializer$$createOrcValue(OrcSerializ > er.scala:226) > at > org.apache.spark.sql.execution.datasources.orc.OrcSerializer.<init>(OrcSerializer.scala:36) > at > org.apache.spark.sql.execution.datasources.orc.OrcOutputWriter.<init>(OrcOutputWriter.scala:36) > at > org.apache.spark.sql.execution.datasources.orc.OrcFileFormat$$anon$1.newInstance(OrcFileFormat.scala:108) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$SingleDirectoryWriteTask.newOutputWriter(FileFormatWriter.scala:376) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$SingleDirectoryWriteTask.execute(FileFormatWriter.scala:387) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask$3.apply > (FileFormatWriter.scala:278) > {code} > It seems to be better to verify a write schema in a driver side for users > along with the CSV fromat; > https://github.com/apache/spark/blob/76ecd095024a658bf68e5db658e4416565b30c17/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVFileFormat.scala#L65 -- This message was sent by Atlassian JIRA (v7.6.3#76005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org