Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/22666#discussion_r228786427
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala
---
@@ -19,14 +19,39 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.util.ArrayBasedMapData
-import org.apache.spark.sql.types.{MapType, StringType, StructType}
+import org.apache.spark.sql.types.{DataType, MapType, StringType,
StructType}
+import org.apache.spark.unsafe.types.UTF8String
object ExprUtils {
- def evalSchemaExpr(exp: Expression): StructType = exp match {
- case Literal(s, StringType) => StructType.fromDDL(s.toString)
+ def evalSchemaExpr(exp: Expression): StructType = {
+ // Use `DataType.fromDDL` since the type string can be struct<...>.
+ val dataType = exp match {
+ case Literal(s, StringType) =>
+ DataType.fromDDL(s.toString)
+ case e @ SchemaOfCsv(_: Literal, _) =>
+ val ddlSchema = e.eval(EmptyRow).asInstanceOf[UTF8String]
+ DataType.fromDDL(ddlSchema.toString)
+ case e => throw new AnalysisException(
+ "Schema should be specified in DDL format as a string literal or
output of " +
+ s"the schema_of_csv function instead of ${e.sql}")
+ }
+
+ if (!dataType.isInstanceOf[StructType]) {
+ throw new AnalysisException(
+ s"Schema should be struct type but got ${dataType.sql}.")
+ }
+ dataType.asInstanceOf[StructType]
+ }
+
+ def evalTypeExpr(exp: Expression): DataType = exp match {
+ case Literal(s, StringType) => DataType.fromDDL(s.toString)
--- End diff --
we also need to update
https://github.com/apache/spark/pull/22666/files#diff-5321c01e95bffc4413c5f3457696213eR157
in case the constant folding rule is disabled.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
