AngersZhuuuu commented on a change in pull request #29085:
URL: https://github.com/apache/spark/pull/29085#discussion_r458712933
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
##########
@@ -754,7 +755,80 @@ class AstBuilder(conf: SQLConf) extends
SqlBaseBaseVisitor[AnyRef] with Logging
outRowFormat: RowFormatContext,
recordReader: Token,
schemaLess: Boolean): ScriptInputOutputSchema = {
- throw new ParseException("Script Transform is not supported", ctx)
+ if (recordWriter != null || recordReader != null) {
+ // TODO: what does this message mean?
+ throw new ParseException(
+ "Unsupported operation: Used defined record reader/writer classes.",
ctx)
+ }
+
+ // Decode and input/output format.
+ type Format = (Seq[(String, String)], Option[String], Seq[(String,
String)], Option[String])
+
+ def format(
+ fmt: RowFormatContext,
+ configKey: String,
+ defaultConfigValue: String): Format = fmt match {
+ case c: RowFormatDelimitedContext =>
+ // TODO we should use the visitRowFormatDelimited function here.
However HiveScriptIOSchema
+ // expects a seq of pairs in which the old parsers' token names are
used as keys.
+ // Transforming the result of visitRowFormatDelimited would be quite a
bit messier than
+ // retrieving the key value pairs ourselves.
+ def entry(key: String, value: Token): Seq[(String, String)] = {
+ Option(value).map(t => key -> t.getText).toSeq
+ }
+
+ val entries = entry("TOK_TABLEROWFORMATFIELD", c.fieldsTerminatedBy) ++
+ entry("TOK_TABLEROWFORMATCOLLITEMS", c.collectionItemsTerminatedBy)
++
+ entry("TOK_TABLEROWFORMATMAPKEYS", c.keysTerminatedBy) ++
+ entry("TOK_TABLEROWFORMATLINES", c.linesSeparatedBy) ++
+ entry("TOK_TABLEROWFORMATNULL", c.nullDefinedAs)
+
+ (entries, None, Seq.empty, None)
+
+ case c: RowFormatContext if
!conf.getConf(CATALOG_IMPLEMENTATION).equals("hive") =>
+ throw new ParseException("TRANSFORM with serde is only supported in
hive mode", ctx)
+
+ case c: RowFormatSerdeContext =>
+ // Use a serde format.
+ val CatalogStorageFormat(None, None, None, Some(name), _, props) =
visitRowFormatSerde(c)
+
+ // SPARK-10310: Special cases LazySimpleSerDe
+ val recordHandler = if (name ==
"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") {
+ Option(conf.getConfString(configKey, defaultConfigValue))
+ } else {
+ None
+ }
+ (Seq.empty, Option(name), props.toSeq, recordHandler)
+
+ case null if conf.getConf(CATALOG_IMPLEMENTATION).equals("hive") =>
Review comment:
> The hive-specific code part should be placed in `SparkSqlParser`.
Could you check related code around again? e.g., `visitCreateTable`.
Done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
