pin_zhang created SPARK-18536: --------------------------------- Summary: Failed to save to hive table when case class with empty field Key: SPARK-18536 URL: https://issues.apache.org/jira/browse/SPARK-18536 Project: Spark Issue Type: Bug Components: SQL Affects Versions: 2.0.1 Reporter: pin_zhang
import scala.collection.mutable.Queue import org.apache.spark.SparkConf import org.apache.spark.SparkContext import org.apache.spark.sql.SaveMode import org.apache.spark.sql.SparkSession import org.apache.spark.streaming.Seconds import org.apache.spark.streaming.StreamingContext 1. Test code case class EmptyC() case class EmptyCTable(dimensions: EmptyC, timebin: java.lang.Long) object EmptyTest { def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("scala").setMaster("local[2]") val ctx = new SparkContext(conf) val spark = SparkSession.builder().enableHiveSupport().config(conf).getOrCreate() val seq = Seq(EmptyCTable(EmptyC(), 1000000L)) val rdd = ctx.makeRDD[EmptyCTable](seq) val ssc = new StreamingContext(ctx, Seconds(1)) val queue = Queue(rdd) val s = ssc.queueStream(queue, false); s.foreachRDD((rdd, time) => { if (!rdd.isEmpty) { import spark.sqlContext.implicits._ rdd.toDF.write.mode(SaveMode.Overwrite).saveAsTable("empty_table") } }) ssc.start() ssc.awaitTermination() } } 2. Exception Caused by: java.lang.IllegalStateException: Cannot build an empty group at org.apache.parquet.Preconditions.checkState(Preconditions.java:91) at org.apache.parquet.schema.Types$GroupBuilder.build(Types.java:554) at org.apache.parquet.schema.Types$GroupBuilder.build(Types.java:426) at org.apache.parquet.schema.Types$Builder.named(Types.java:228) at org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaConverter.convertField(ParquetSchemaConverter.scala:527) at org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaConverter.convertField(ParquetSchemaConverter.scala:321) at org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaConverter$$anonfun$convert$1.apply(ParquetSchemaConverter.scala:313) at org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaConverter$$anonfun$convert$1.apply(ParquetSchemaConverter.scala:313) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.Iterator$class.foreach(Iterator.scala:893) at scala.collection.AbstractIterator.foreach(Iterator.scala:1336) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at org.apache.spark.sql.types.StructType.foreach(StructType.scala:95) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at org.apache.spark.sql.types.StructType.map(StructType.scala:95) at org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaConverter.convert(ParquetSchemaConverter.scala:313) at org.apache.spark.sql.execution.datasources.parquet.ParquetWriteSupport.init(ParquetWriteSupport.scala:85) at org.apache.parquet.hadoop.ParquetOutputFormat.getRecordWriter(ParquetOutputFormat.java:288) at org.apache.parquet.hadoop.ParquetOutputFormat.getRecordWriter(ParquetOutputFormat.java:262) at org.apache.spark.sql.execution.datasources.parquet.ParquetOutputWriter.<init>(ParquetFileFormat.scala:562) at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat$$anon$1.newInstance(ParquetFileFormat.scala:139) at org.apache.spark.sql.execution.datasources.BaseWriterContainer.newOutputWriter(WriterContainer.scala:131) at org.apache.spark.sql.execution.datasources.DefaultWriterContainer.writeRows(WriterContainer.scala:247) at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(InsertIntoHadoopFsRelationCommand.scala:143) at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(InsertIntoHadoopFsRelationCommand.scala:143) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70) at org.apache.spark.scheduler.Task.run(Task.scala:86) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) ... 3 more -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org