[ https://issues.apache.org/jira/browse/SPARK-24681?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Adrian Ionescu updated SPARK-24681: ----------------------------------- Description: Here's a patch that reproduces the issue: {code:java} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala index 09c1547..29bb3db 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hive import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.execution.datasources.parquet.ParquetTest +import org.apache.spark.sql.functions.{lit, struct} import org.apache.spark.sql.hive.test.TestHiveSingleton case class Cases(lower: String, UPPER: String) @@ -76,4 +77,21 @@ class HiveParquetSuite extends QueryTest with ParquetTest with TestHiveSingleton } } } + + test("column names including ':' characters") { + withTempPath { path => + withTable("test_table") { + spark.range(0) + .select(struct(lit(0).as("nested:column")).as("toplevel:column")) + .write.format("parquet") + .option("path", path.getCanonicalPath) + .saveAsTable("test_table") + + sql("CREATE VIEW test_view_1 AS SELECT `toplevel:column`.* FROM test_table") + sql("CREATE VIEW test_view_2 AS SELECT * FROM test_table") + + } + } + } }{code} The first "CREATE VIEW" statement succeeds, but the second one fails with: {code:java} org.apache.spark.SparkException: Cannot recognize hive type string: struct<nested:column:int> {code} was: Here's a patch that reproduces the issue: {code:java} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala index 09c1547..29bb3db 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hive import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.execution.datasources.parquet.ParquetTest +import org.apache.spark.sql.functions.{lit, struct} import org.apache.spark.sql.hive.test.TestHiveSingleton case class Cases(lower: String, UPPER: String) @@ -76,4 +77,21 @@ class HiveParquetSuite extends QueryTest with ParquetTest with TestHiveSingleton } } } + + test("column names including ':' characters") { + withTempPath { path => + withTable("test_table") { + spark.range(0) + .select(struct(lit(0).as("nested:column")).as("toplevel:column")) + .write.format("parquet") + .option("path", path.getCanonicalPath) + .saveAsTable("test_table") + + sql("CREATE VIEW test_view_1 AS SELECT `toplevel:column`.* FROM test_table") + sql("CREATE VIEW test_view_2 AS SELECT * FROM test_table") + + } + } + } }{code} The last sql statement in there fails with: {code:java} org.apache.spark.SparkException: Cannot recognize hive type string: struct<nested:column:int> {code} > Cannot create a view from a table when a nested column name contains ':' > ------------------------------------------------------------------------ > > Key: SPARK-24681 > URL: https://issues.apache.org/jira/browse/SPARK-24681 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 2.2.0, 2.3.0, 2.4.0 > Reporter: Adrian Ionescu > Priority: Major > > Here's a patch that reproduces the issue: > {code:java} > diff --git > a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala > b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala > index 09c1547..29bb3db 100644 > --- > a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala > +++ > b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala > @@ -19,6 +19,7 @@ package org.apache.spark.sql.hive > > import org.apache.spark.sql.{QueryTest, Row} > import org.apache.spark.sql.execution.datasources.parquet.ParquetTest > +import org.apache.spark.sql.functions.{lit, struct} > import org.apache.spark.sql.hive.test.TestHiveSingleton > > case class Cases(lower: String, UPPER: String) > @@ -76,4 +77,21 @@ class HiveParquetSuite extends QueryTest with ParquetTest > with TestHiveSingleton > } > } > } > + > + test("column names including ':' characters") { > + withTempPath { path => > + withTable("test_table") { > + spark.range(0) > + .select(struct(lit(0).as("nested:column")).as("toplevel:column")) > + .write.format("parquet") > + .option("path", path.getCanonicalPath) > + .saveAsTable("test_table") > + > + sql("CREATE VIEW test_view_1 AS SELECT `toplevel:column`.* FROM > test_table") > + sql("CREATE VIEW test_view_2 AS SELECT * FROM test_table") > + > + } > + } > + } > }{code} > The first "CREATE VIEW" statement succeeds, but the second one fails with: > {code:java} > org.apache.spark.SparkException: Cannot recognize hive type string: > struct<nested:column:int> > {code} -- This message was sent by Atlassian JIRA (v7.6.3#76005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org