AngersZhuuuu commented on a change in pull request #29085:
URL: https://github.com/apache/spark/pull/29085#discussion_r458778858
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
##########
@@ -679,64 +680,78 @@ class SparkSqlAstBuilder(conf: SQLConf) extends
AstBuilder(conf) {
"Unsupported operation: Used defined record reader/writer classes.",
ctx)
}
- // Decode and input/output format.
- type Format = (Seq[(String, String)], Option[String], Seq[(String,
String)], Option[String])
- def format(
- fmt: RowFormatContext,
- configKey: String,
- defaultConfigValue: String): Format = fmt match {
- case c: RowFormatDelimitedContext =>
- // TODO we should use the visitRowFormatDelimited function here.
However HiveScriptIOSchema
- // expects a seq of pairs in which the old parsers' token names are
used as keys.
- // Transforming the result of visitRowFormatDelimited would be quite a
bit messier than
- // retrieving the key value pairs ourselves.
- def entry(key: String, value: Token): Seq[(String, String)] = {
- Option(value).map(t => key -> t.getText).toSeq
- }
- val entries = entry("TOK_TABLEROWFORMATFIELD", c.fieldsTerminatedBy) ++
- entry("TOK_TABLEROWFORMATCOLLITEMS", c.collectionItemsTerminatedBy)
++
- entry("TOK_TABLEROWFORMATMAPKEYS", c.keysTerminatedBy) ++
- entry("TOK_TABLEROWFORMATLINES", c.linesSeparatedBy) ++
- entry("TOK_TABLEROWFORMATNULL", c.nullDefinedAs)
-
- (entries, None, Seq.empty, None)
+ if (!conf.getConf(CATALOG_IMPLEMENTATION).equals("hive")) {
+ super.withScriptIOSchema(
+ ctx,
+ inRowFormat,
+ recordWriter,
+ outRowFormat,
+ recordReader,
+ schemaLess)
+ } else {
- case c: RowFormatSerdeContext =>
- // Use a serde format.
- val CatalogStorageFormat(None, None, None, Some(name), _, props) =
visitRowFormatSerde(c)
+ // Decode and input/output format.
+ type Format = (Seq[(String, String)], Option[String], Seq[(String,
String)], Option[String])
+
+ def format(
+ fmt: RowFormatContext,
+ configKey: String,
+ defaultConfigValue: String): Format = fmt match {
+ case c: RowFormatDelimitedContext =>
+ // TODO we should use visitRowFormatDelimited function here. However
HiveScriptIOSchema
+ // expects a seq of pairs in which the old parsers' token names are
used as keys.
+ // Transforming the result of visitRowFormatDelimited would be quite
a bit messier than
+ // retrieving the key value pairs ourselves.
+ def entry(key: String, value: Token): Seq[(String, String)] = {
+ Option(value).map(t => key -> t.getText).toSeq
+ }
+
+ val entries = entry("TOK_TABLEROWFORMATFIELD", c.fieldsTerminatedBy)
++
+ entry("TOK_TABLEROWFORMATCOLLITEMS",
c.collectionItemsTerminatedBy) ++
+ entry("TOK_TABLEROWFORMATMAPKEYS", c.keysTerminatedBy) ++
+ entry("TOK_TABLEROWFORMATLINES", c.linesSeparatedBy) ++
+ entry("TOK_TABLEROWFORMATNULL", c.nullDefinedAs)
Review comment:
> We cannot share this code part with `AstBuilder` one?
No, since hive can define just input as serde or just output as serde, so
we need to keep this both part.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
