[ https://issues.apache.org/jira/browse/SPARK-18058?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15596978#comment-15596978 ]
Muthu Jayakumar edited comment on SPARK-18058 at 10/22/16 2:40 AM: ------------------------------------------------------------------- Wanted to share the test I used while getting this error from Spark 2.0.0 Schema from parquet file. {code} scala> d1.printSchema() root |-- task_id: string (nullable = true) |-- task_name: string (nullable = true) |-- some_histogram: struct (nullable = true) | |-- values: array (nullable = true) | | |-- element: double (containsNull = true) | |-- freq: array (nullable = true) | | |-- element: long (containsNull = true) scala> d2.printSchema() //Data created using dataframe and/or processed before writing to parquet file. root |-- task_id: string (nullable = true) |-- task_name: string (nullable = true) |-- some_histogram: struct (nullable = true) | |-- values: array (nullable = true) | | |-- element: double (containsNull = false) | |-- freq: array (nullable = true) | | |-- element: long (containsNull = false) scala> d1.union(d2).printSchema() Exception in thread "main" org.apache.spark.sql.AnalysisException: unresolved operator 'Union; at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:40) at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:58) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:361) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:67) at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:67) at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:58) at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:49) at org.apache.spark.sql.Dataset.<init>(Dataset.scala:161) at org.apache.spark.sql.Dataset.<init>(Dataset.scala:167) at org.apache.spark.sql.Dataset$.apply(Dataset.scala:59) at org.apache.spark.sql.Dataset.withTypedPlan(Dataset.scala:2594) at org.apache.spark.sql.Dataset.union(Dataset.scala:1459) {code} But the same union like operation works when I use the following schema {code} scala> df1.printSchema() root |-- col1: integer (nullable = false) |-- col2: string (nullable = true) |-- col3: double (nullable = false) |-- col4: string (nullable = false) scala> df2.printSchema() root |-- col1: integer (nullable = true) |-- col2: string (nullable = true) |-- col3: double (nullable = true) |-- col4: string (nullable = true) scala> d1.union(d2).printSchema() //this one worked. root |-- col1: integer (nullable = true) |-- col2: string (nullable = true) |-- col3: double (nullable = true) |-- col4: string (nullable = true) {code} was (Author: babloo80): Wanted to share the test I used while getting this error from Spark 2.0.0 Schema from parquet file. {code} d1.printSchema() root |-- task_id: string (nullable = true) |-- task_name: string (nullable = true) |-- some_histogram: struct (nullable = true) | |-- values: array (nullable = true) | | |-- element: double (containsNull = true) | |-- freq: array (nullable = true) | | |-- element: long (containsNull = true) d2.printSchema() //Data created using dataframe and/or processed before writing to parquet file. root |-- task_id: string (nullable = true) |-- task_name: string (nullable = true) |-- some_histogram: struct (nullable = true) | |-- values: array (nullable = true) | | |-- element: double (containsNull = false) | |-- freq: array (nullable = true) | | |-- element: long (containsNull = false) d1.union(d2).printSchema() Exception in thread "main" org.apache.spark.sql.AnalysisException: unresolved operator 'Union; at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:40) at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:58) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:361) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:67) at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:67) at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:58) at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:49) at org.apache.spark.sql.Dataset.<init>(Dataset.scala:161) at org.apache.spark.sql.Dataset.<init>(Dataset.scala:167) at org.apache.spark.sql.Dataset$.apply(Dataset.scala:59) at org.apache.spark.sql.Dataset.withTypedPlan(Dataset.scala:2594) at org.apache.spark.sql.Dataset.union(Dataset.scala:1459) {code} > AnalysisException may be thrown when union two DFs whose struct fields have > different nullability > ------------------------------------------------------------------------------------------------- > > Key: SPARK-18058 > URL: https://issues.apache.org/jira/browse/SPARK-18058 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 1.6.2, 2.0.1 > Reporter: Cheng Lian > > The following Spark shell snippet reproduces this issue: > {code} > spark.range(10).createOrReplaceTempView("t1") > spark.range(10).map(i => i: > java.lang.Long).toDF("id").createOrReplaceTempView("t2") > sql("SELECT struct(id) FROM t1 UNION ALL SELECT struct(id) FROM t2") > {code} > {noformat} > org.apache.spark.sql.AnalysisException: Union can only be performed on tables > with the compatible column types. StructType(StructField(id,LongType,true)) > <> StructType(StructField(id,LongType,false)) at the first column of the > second table; > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:40) > at > org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:57) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$11$$anonfun$apply$12.apply(CheckAnalysis.scala:291) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$11$$anonfun$apply$12.apply(CheckAnalysis.scala:289) > at scala.collection.immutable.List.foreach(List.scala:381) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$11.apply(CheckAnalysis.scala:289) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$11.apply(CheckAnalysis.scala:278) > at scala.collection.immutable.List.foreach(List.scala:381) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:278) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:67) > at > org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:132) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:67) > at > org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:57) > at > org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:49) > at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:61) > at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:573) > ... 50 elided > {noformat} > The reason is that we treat two {{StructType}} incompatible even if their > only differ from each other in field nullability. -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org