ganeshashree commented on code in PR #59828:
URL: https://github.com/apache/spark/pull/59828#discussion_r4439670973
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala:
##########
@@ -2594,18 +2620,44 @@ class Analyzer(
case q: LogicalPlan =>
q.transformExpressionsUpWithPruning(
- _.containsAnyPattern(UNRESOLVED_FUNCTION, GENERATOR),
+ _.containsAnyPattern(UNRESOLVED_FUNCTION, GENERATOR,
UNRESOLVED_JSON_CONSTRUCTOR),
ruleId) {
- case u @ UnresolvedFunction(nameParts, arguments, _, _, _, _, _, _)
- if functionResolution.hasLambdaAndResolvedArguments(arguments)
=> withPosition(u) {
- functionResolution.resolveFunction(u) match {
- case func: HigherOrderFunction => func
- case other => other.failAnalysis(
- errorClass =
"INVALID_LAMBDA_FUNCTION_CALL.NON_HIGHER_ORDER_FUNCTION",
- messageParameters = Map(
- "class" -> other.getClass.getCanonicalName))
+ case u @ UnresolvedFunction(_, arguments, _, _, _, _, _, _)
+ if functionResolution.hasLambdaAndResolvedArguments(arguments) =>
+ resolveHigherOrderFunction(u)
+
+ // Handle JSON constructors in this same bottom-up pass, so nesting
them inside other
+ // calls does not cost extra analyzer iterations.
+ case j: UnresolvedJsonConstructor
+ if functionResolution.routesJsonConstructorToFunction(j) =>
+ // A routine owns the name: resolve like the equivalent flat call.
+ val u = j.toUnresolvedFunction
+ if (functionResolution.hasLambdaAndResolvedArguments(u.arguments))
{
+ resolveHigherOrderFunction(u)
+ } else if (!u.childrenResolved) {
+ u
+ } else {
+ functionResolution.resolveFunction(u)
+ }
+
+ case j: UnresolvedJsonConstructor =>
+ val directStars = j.arguments.collect { case s: Star => s }
+ if (directStars.nonEmpty) {
+ throw QueryCompilationErrors.invalidStarUsageError(
+ s"expression `${j.prettyName}`", directStars)
+ }
+ // The built-in owns it. Switch to the built-in form so the owner
is checked only once.
+ val stock = j match {
+ case r: RoutedJsonConstructor => r.toStock
+ case s: StockJsonConstructor => s
+ }
+ if (!stock.childrenResolved) {
Review Comment:
Done.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -1314,6 +1311,46 @@ trait ImplicitlyFormattedAsJson extends Expression {
def emitsImplicitJsonText: Boolean = true
}
+object ImplicitlyFormattedAsJson {
+ /**
+ * Whether `e` emits JSON text, so a JSON constructor splices it instead of
quoting it. Sees
+ * through `Collate`, which only changes the collation, but not `Cast`:
`CAST(... AS STRING)` is
+ * how a user (and `JsonArray.sql`) asks for the value to be quoted.
+ */
+ @scala.annotation.tailrec
+ def isImplicitlyJson(e: Expression): Boolean = e match {
+ case c: Collate => isImplicitlyJson(c.child)
+ case i: ImplicitlyFormattedAsJson => i.emitsImplicitJsonText
+ case _ => false
+ }
+
+ /**
+ * The splice decision for a resolved argument, used by the builders and by
the parser wrappers
+ * when they finalize their flags. Like [[isImplicitlyJson]], but also sees
through the collation
+ * `Cast` that the analyzer adds under an object or view default collation
(mixed with `Collate`
+ * in any order). A user cast, including `string(...)`, still makes the
value quoted.
+ */
+ @scala.annotation.tailrec
+ def splicesThroughCollationCast(e: Expression): Boolean = e match {
Review Comment:
Done.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala:
##########
@@ -941,11 +941,10 @@ object FunctionRegistry {
expression[StructsToCsv]("to_csv")
)
- // Built-in forms of the SQL:2016 JSON constructor and path functions.
`AstBuilder` routes the
- // eligible flat clause-free calls here so a same-named routine can shadow
them. Clause-bearing
- // forms, and the nested/implicit-JSON clause-free forms (routing deferred
to SPARK-59243), are
- // still built directly by the grammar -- except the `json_object` comma
form, whose nested
- // producer value also routes here (carrying a `JsonImplicitFormatCarrier`).
+ // Built-in forms of the SQL:2016 JSON constructor and path functions.
`AstBuilder` routes
+ // clause-free calls here so a same-named routine can shadow them; a
nested-constructor call
+ // routes as a `RoutedJsonConstructor`. Calls with a clause or explicit
FORMAT JSON, and calls
+ // written directly as another constructor's value, always mean the built-in.
Review Comment:
Done.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -1720,6 +1740,145 @@ object JsonArray {
}
}
+/**
+ * A parsed `JSON_ARRAY(...)` / `JSON_OBJECT(...)` that keeps the parser's
splice flags until its
+ * children resolve; the resolved children can then add splices the parser
could not see (e.g. a
+ * qualified `builtin.json_array(...)` argument).
+ *
+ * The built-in owns a [[StockJsonConstructor]]. A [[RoutedJsonConstructor]]
can be owned by a
+ * same-named routine or an `injectFunction` replacement, and then resolves as
an ordinary call.
+ */
+sealed trait UnresolvedJsonConstructor extends Unevaluable with
ImplicitlyFormattedAsJson {
+ def routeThroughResolution: Boolean
+ /** The arguments a routine owning the call receives. */
+ def arguments: Seq[Expression] = children
+ /** The built-in expression, with the splice flags finalized. */
+ def toBuiltin: TimeZoneAwareExpression
+ /** The ordinary call this stands for when a routine owns the name. */
+ def toUnresolvedFunction: UnresolvedFunction =
CurrentOrigin.withOrigin(origin) {
+ UnresolvedFunction(prettyName, arguments, isDistinct = false)
+ }
+
+ override def dataType: DataType = throw new UnresolvedException("dataType")
+ override def nullable: Boolean = throw new UnresolvedException("nullable")
+ override lazy val resolved: Boolean = false
+ final override val nodePatterns: Seq[TreePattern] =
Seq(UNRESOLVED_JSON_CONSTRUCTOR)
+}
+
+/**
+ * An [[UnresolvedJsonConstructor]] the built-in owns: a call with a clause or
explicit
+ * `FORMAT JSON`, a call written directly as another constructor's value (the
inner `JSON_ARRAY(1)`
+ * in `JSON_ARRAY(JSON_ARRAY(1))`), or a routed call the built-in turned out
to own.
+ */
+sealed trait StockJsonConstructor
+ extends UnresolvedJsonConstructor
+ with DefaultStringProducingExpression {
+ def returning: DataType
+ final override def routeThroughResolution: Boolean = false
+ // Report the RETURNING type, so an explicit `RETURNING STRING COLLATE c` is
not recolored to the
+ // object or view default collation.
+ override def dataType: DataType = returning
+
+ // Keep every parser flag and add a splice for each resolved child that is a
JSON producer. Such a
+ // child is a trusted constructor, so it never needs validation.
+ protected def finalizeFlags(
+ parserFlags: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ values: Seq[Expression]): (Seq[Boolean], Seq[Boolean]) = {
+ val splices =
values.map(ImplicitlyFormattedAsJson.splicesThroughCollationCast)
+ (parserFlags.lazyZip(splices).map(_ || _),
needsValidation.lazyZip(splices).map(_ && !_))
+ }
+}
+
+/**
+ * A clause-free [[UnresolvedJsonConstructor]] whose owner is decided during
resolution. Not a
+ * `DefaultStringProducingExpression`: until the owner is known it stands for
an ordinary call, so
+ * default-collation rules must not cast it (a shadow keeps its own result
type).
+ */
+sealed trait RoutedJsonConstructor extends UnresolvedJsonConstructor {
+ final override def routeThroughResolution: Boolean = true
+ /** The built-in-owned form with the clause-free defaults, once the built-in
owns the call. */
+ def toStock: StockJsonConstructor
+ override def toBuiltin: TimeZoneAwareExpression = toStock.toBuiltin
+}
+
+case class UnresolvedJsonArray(
+ values: Seq[Expression],
+ formatJson: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ nullBehavior: JsonConstructorNullBehavior,
+ returning: DataType)
+ extends StockJsonConstructor {
+ assert(values.length == formatJson.length && values.length ==
needsValidation.length,
+ "UnresolvedJsonArray requires one formatJson and one needsValidation flag
per value")
+ assert(needsValidation.lazyZip(formatJson).forall((nv, fj) => !nv || fj),
+ "UnresolvedJsonArray needsValidation implies formatJson")
+ override def children: Seq[Expression] = values
+ override def prettyName: String = "json_array"
+ override protected def withNewChildrenInternal(
+ newChildren: IndexedSeq[Expression]): UnresolvedJsonArray = copy(values
= newChildren)
+
+ override def toBuiltin: JsonArray = {
+ val (finalFormatJson, finalNeedsValidation) = finalizeFlags(formatJson,
needsValidation, values)
+ JsonArray(values, finalFormatJson, finalNeedsValidation, nullBehavior,
returning)
+ }
+}
+
+/** A clause-free `JSON_ARRAY(...)` with a nested constructor argument. */
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
