ganeshashree commented on code in PR #58034:
URL: https://github.com/apache/spark/pull/58034#discussion_r4132952956
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -1283,17 +1284,17 @@ object JsonConstructorNullBehavior {
/**
* Marker for expressions whose result is JSON text and therefore carry an
implicit SQL/JSON
* `FORMAT JSON`: when such an expression appears as an argument of a JSON
constructor (e.g.
- * `JSON_ARRAY`), its value is spliced in verbatim rather than quoted as a
JSON string:
+ * `JSON_ARRAY` or `JSON_OBJECT`), its value is spliced in verbatim rather
than quoted as a JSON
Review Comment:
Done.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -1745,6 +1746,59 @@ object JsonArrayExpressionBuilder extends
ExpressionBuilder {
}
}
+/**
+ * Marks a routed comma-form `JSON_OBJECT(...)` value as a lexically nested
JSON producer (implicit
+ * `FORMAT JSON`). Transparent (keeps its child's type, value, foldability),
so it does not perturb
+ * a shadowing routine's overload selection; the unshadowed built-in unwraps
it and splices raw.
+ * Attached only by direct `JSON_OBJECT(...)` syntax, so qualified/generic
calls keep quoting
+ * pending SPARK-59243.
+ */
+case class JsonImplicitFormatCarrier(child: Expression) extends
TaggingExpression {
+ // Render as the bare child so generated SQL for a carrier that survives
into a routine's plan
+ // stays reparseable (the built-in unwraps it before rendering its own SQL).
+ override def sql: String = child.sql
+ override protected def withNewChildInternal(newChild: Expression):
JsonImplicitFormatCarrier =
+ copy(child = newChild)
+}
+
+@ExpressionDescription(
+ usage =
+ "_FUNC_([key, value [, key, value ...]]) - Returns a JSON object string
built from the " +
+ "given key/value pairs.",
+ arguments = """
+ Arguments:
+ * key - a non-null string naming an object member.
+ * value - the member's value.
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_('id', 7, 'name', 'Ada');
+ {"id":7,"name":"Ada"}
+ > SELECT _FUNC_();
+ {}
+ """,
+ since = "4.4.0",
+ group = "json_funcs")
+object JsonObjectExpressionBuilder extends ExpressionBuilder {
+ override def build(funcName: String, expressions: Seq[Expression]):
Expression = {
+ // The routed form is a flat `key, value, ...` list (the MySQL-style
spelling), so it must have
+ // an even length; an odd count leaves a dangling key with no value.
+ if (expressions.length % 2 != 0) {
+ throw QueryCompilationErrors.wrongNumArgsError(
+ funcName, Seq("2n (n >= 0)"), expressions.length)
+ }
+ // A `JsonImplicitFormatCarrier` marks a lexically nested JSON producer,
spliced raw (unwrapped
+ // here) and trusted; every other value is quoted. Routed calls carry no
explicit FORMAT JSON,
Review Comment:
Done.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -2272,3 +2326,409 @@ case class JsonTypeof(child: Expression)
override protected def withNewChildInternal(newChild: Expression):
JsonTypeof =
copy(child = newChild)
}
+
+/**
+ * The SQL:2016 `JSON_OBJECT` constructor function (feature T811): constructs
a JSON object from
+ * key-value pairs, written `key VALUE value`, `KEY key VALUE value`, `key :
value`, or the
+ * MySQL-style `key, value`.
+ *
+ * Keys must be non-null strings; a null key is an error. The `ON NULL` clause
controls whether
+ * null-valued pairs are included (NULL ON NULL, the standard default) or
omitted (ABSENT ON NULL).
+ *
+ * A value is spliced in raw (unquoted) when it carries `FORMAT JSON` (tracked
per member by
+ * `rawJson`): the explicit clause (e.g. `JSON_OBJECT('a' VALUE '{"b":1}'
FORMAT JSON)`), or the
+ * implicit form carried by a nested JSON constructor / `JSON_QUERY` under
KEEP QUOTES (see
+ * `rawJsonValue`). Explicit `FORMAT JSON` on a non-constructor is arbitrary
user text, so
+ * `needsValidation` marks it for JSON validation at eval; a nested
constructor is trusted.
+ *
+ * Examples:
+ * JSON_OBJECT('id' VALUE 7, 'name' VALUE 'Ada') ->
'{"id":7,"name":"Ada"}'
+ * JSON_OBJECT('id': 7, 'v': NULL) -> '{"id":7,"v":null}'
+ * JSON_OBJECT('id': 7, 'v': NULL ABSENT ON NULL) -> '{"id":7}'
+ * JSON_OBJECT() -> '{}'
+ *
+ * A clause-free call that is not itself lexically nested in an enclosing
constructor's *value*
+ * position routes through function resolution and is rebuilt by
`JsonObjectExpressionBuilder`, so a
+ * same-named routine can shadow the built-in `json_object`: always for the
comma form (a nested
+ * producer value routes too, carrying its raw-splice eligibility via
`JsonImplicitFormatCarrier`),
+ * but for the `VALUE` / `:` forms only when no member carries a raw value
(`rawJson.contains(true)`
+ * -- an explicit `FORMAT JSON` or a nested producer in *value* position --
keeps it direct).
+ * Otherwise it is built directly from the grammar (clause-bearing, or
value-nested); a nested
+ * *key*-position call stays routable (see `AstBuilder.visitJsonObject`).
+ * The user-facing reference lives in
`docs/sql-ref-syntax-qry-select-json-object.md`.
+ */
+case class JsonObjectExpr(
+ members: Seq[(Expression, Expression)],
+ rawJson: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ nullBehavior: JsonConstructorNullBehavior =
JsonConstructorNullBehavior.Null,
+ returning: DataType = StringType,
+ timeZoneId: Option[String] = None)
+ extends Expression
+ with TimeZoneAwareExpression
+ with CodegenFallback
+ with QueryErrorsBase
+ // Default RETURNING is a plain STRING, so
`DefaultStringProducingExpression` lets
+ // `ApplyDefaultCollation` cast the result to a non-default collation; the
`dataType` override
+ // below stays authoritative when RETURNING is given explicitly.
+ with DefaultStringProducingExpression
+ with ImplicitlyFormattedAsJson {
+
+ // `rawJson(i)` splices member `i`'s value verbatim (already-JSON text)
rather than quoting it;
+ // `needsValidation(i)` marks that raw text as arbitrary user input to
JSON-validate at eval (an
+ // explicit `FORMAT JSON` on a non-constructor). Both are frozen at parse
time (see
+ // `AstBuilder.visitJsonObject`). A validated value is always a spliced one.
+ require(members.length == rawJson.length && members.length ==
needsValidation.length,
+ "JsonObjectExpr requires one rawJson and one needsValidation flag per
member")
+ require(needsValidation.lazyZip(rawJson).forall((nv, raw) => !nv || raw),
+ "JsonObjectExpr needsValidation implies rawJson")
+
+ @transient private lazy val memberArray: Array[(Expression, Expression)] =
+ members.toArray
+
+ @transient private lazy val rawJsonArray: Array[Boolean] =
+ rawJson.toArray
+
+ @transient private lazy val needsValidationArray: Array[Boolean] =
+ needsValidation.toArray
+
+ // Always throwable: a null key raises JSON_OBJECT_NULL_KEY at eval even
with non-throwable
+ // children, which keeps the optimizer from pushing it below a filtering
join. Left non-foldable
+ // (the default) so folding a constant call does not eagerly raise that
null-key error at
+ // optimization for rows a filter would later drop.
+ override lazy val throwable: Boolean = true
+
+ // The value is never null, but report nullable as `throwable` (like
JsonArray): `NullPropagation`
+ // keys off `nullable`, so a non-nullable constructor would let it fold `IS
[NOT] NULL` to a
+ // constant or `count(...)` to `count(1)`, skipping the eval that must raise
JSON_OBJECT_NULL_KEY.
+ override def nullable: Boolean = throwable
+
+ override def dataType: DataType = returning
+
+ // Flattened `[k0, v0, k1, v1, ...]` view, cached since `children` is walked
repeatedly.
+ @transient private lazy val childrenSeq: Seq[Expression] =
+ memberArray.flatMap { case (k, v) => Seq(k, v) }.toImmutableArraySeq
+
+ override def children: Seq[Expression] = childrenSeq
+
+ override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+ copy(timeZoneId = Option(timeZoneId))
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ val inputCheck = super.checkInputDataTypes()
+ if (inputCheck.isFailure) {
+ inputCheck
+ } else if (!JsonObjectExpr.isValidReturningType(returning)) {
+ DataTypeMismatch(
+ errorSubClass = "INVALID_JSON_RETURNING_TYPE",
+ messageParameters = Map(
+ "functionName" -> toSQLId(prettyName),
+ "returningType" -> toSQLType(returning)))
+ } else {
+ // Keys must be character strings (they become JSON object member
names). Reject any other
+ // concrete type up front so a non-string key cannot reach the
StringType Jackson writer and
+ // fail with an internal ClassCastException. NullType is allowed
through: a null key value is
+ // reported at runtime as JSON_OBJECT_NULL_KEY rather than a type error.
+ val keyCheck = memberArray.iterator.map(_._1).zipWithIndex.collectFirst {
+ case (k, keyIndex) if !(k.dataType.isInstanceOf[StringType] ||
k.dataType == NullType) =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> ordinalNumber(keyIndex * 2),
+ "requiredType" -> toSQLType(StringType),
+ "inputSql" -> toSQLExpr(k),
+ "inputType" -> toSQLType(k.dataType)))
+ }.getOrElse(TypeCheckResult.TypeCheckSuccess)
+ if (keyCheck.isFailure) {
+ keyCheck
+ } else {
+ // Every value must be serializable to JSON. This mirrors `to_json`'s
analysis-time
+ // `JacksonUtils.verifyType` check, plus a guard for the spatial
atomics that check accepts
+ // but `JacksonGenerator` cannot write (rejecting a runtime failure at
analysis instead).
+ memberArray.iterator.map(_._2).zipWithIndex
+ .foldLeft(TypeCheckResult.TypeCheckSuccess: TypeCheckResult) {
+ case (acc, _) if acc.isFailure => acc
+ case (_, (v, valueIndex)) if rawJsonArray(valueIndex) &&
+ !(v.dataType.isInstanceOf[StringType] || v.dataType ==
NullType) =>
+ // A raw-spliced value must carry JSON text (string); NullType
is allowed (eval drops
+ // or emits it before splicing). Reject any other type up front,
matching JSON_ARRAY's
+ // FORMAT JSON check, rather than failing with a
ClassCastException at eval.
+ DataTypeMismatch(
+ errorSubClass = "INVALID_JSON_FORMAT_JSON_INPUT",
+ messageParameters = Map(
+ "functionName" -> toSQLId(prettyName),
+ // Flattened 1-based arg position, not the member ordinal
(value of member i is
+ // arg 2i+2), matching the key check's
`ordinalNumber(keyIndex*2)`.
+ "position" -> (valueIndex * 2 + 2).toString,
+ "inputType" -> toSQLType(v.dataType)))
+ case (_, (v, _)) =>
+ val elemCheck = JacksonUtils.verifyType(prettyName, v.dataType)
+ if (elemCheck.isFailure) {
+ elemCheck
+ } else if
(JsonObjectExpr.containsUnsupportedJsonType(v.dataType)) {
+ DataTypeMismatch(
+ errorSubClass = "CANNOT_CONVERT_TO_JSON",
+ messageParameters = Map(
+ "name" -> toSQLId(prettyName),
+ "type" -> toSQLType(v.dataType)))
+ } else {
+ TypeCheckResult.TypeCheckSuccess
+ }
+ }
+ }
+ }
+ }
+
+ override def stateful: Boolean = true
+
+ @transient private lazy val resolvedZoneId: String =
+ timeZoneId.getOrElse(SQLConf.get.sessionLocalTimeZone)
+
+ // JacksonGenerator (shared with `to_json`, so rendering matches) can only
serialize a container,
+ // not a bare scalar, so each key/value is wrapped in a one-element array
whose brackets are
+ // stripped in renderKey/appendRenderedValue below.
+ @transient private lazy val keyEvaluator: StructsToJsonEvaluator =
+ StructsToJsonEvaluator(Map.empty, ArrayType(StringType),
Some(resolvedZoneId))
+
+ @transient private lazy val valueEvaluators: Array[StructsToJsonEvaluator] =
+ memberArray.map { case (_, v) =>
+ StructsToJsonEvaluator(Map.empty, ArrayType(v.dataType),
+ Some(resolvedZoneId))
+ }
+
+ @transient private lazy val singleElem: Array[Any] = new Array[Any](1)
+
+ @transient private lazy val singleElemData: GenericArrayData =
+ new GenericArrayData(singleElem)
+
+ @transient private lazy val castInput: GenericInternalRow =
+ new GenericInternalRow(1)
+
+ @transient private lazy val returningCast: Expression =
+ Cast(BoundReference(0, StringType, nullable = true), returning, timeZoneId,
+ EvalMode.ANSI)
+
+ // `evaluateString` returns the Java String directly, skipping `evaluate`'s
UTF8String round trip.
+ // The key is cached or appended whole, so it still materializes the
interior substring.
+ private def renderKey(key: Any): String = {
+ singleElem(0) = key
+ val arrJson = keyEvaluator.evaluateString(singleElemData)
+ arrJson.substring(1, arrJson.length - 1)
+ }
+
+ // Append the interior of the "[<frag>]" wrapper straight into the builder
(no per-value
+ // substring), mirroring `JsonArray.appendRenderedElement`.
+ private def appendRenderedValue(sb: java.lang.StringBuilder, idx: Int,
value: Any): Unit = {
+ singleElem(0) = value
+ val arrJson = valueEvaluators(idx).evaluateString(singleElemData)
+ sb.append(arrJson, 1, arrJson.length - 1)
+ }
+
+ @transient private lazy val formatJsonFactory: JsonFactory = new
JsonFactory()
+
+ @transient private lazy val cachedValidatedTexts: Array[String] =
+ new Array[String](memberArray.length)
+
+ // A `FORMAT JSON` value is spliced verbatim, so it must be exactly one
well-formed JSON value
+ // (a string like `1,2` or `{bad` would corrupt the object). Parse one
value; reject anything
+ // trailing. Mirrors `JsonArray.validateJsonText`.
+ private def validateJsonText(idx: Int, text: String): Unit = {
+ val valid =
+ try {
+ Utils.tryWithResource(formatJsonFactory.createParser(text)) { parser =>
+ if (parser.nextToken() == null) {
+ false
+ } else {
+ parser.skipChildren()
+ parser.nextToken() == null
+ }
+ }
+ } catch {
+ case _: JsonProcessingException => false
+ }
+ if (!valid) {
+ // Flattened 1-based value-arg position (member `idx` is arg `2*idx+2`),
matching the
+ // analysis-time INVALID_JSON_FORMAT_JSON_INPUT check.
+ throw QueryExecutionErrors.invalidJsonFormatJsonValueError(prettyName,
idx * 2 + 2, text)
+ }
+ }
+
+ // Raw text to splice for member `idx`: a trusted nested constructor splices
as-is; an explicit
+ // `FORMAT JSON` string is validated once (cached) when foldable, else per
row.
+ private def rawJsonText(idx: Int, value: Any): String = {
+ if (!needsValidationArray(idx)) {
+ value.asInstanceOf[UTF8String].toString
+ } else if (memberArray(idx)._2.foldable) {
+ val cached = cachedValidatedTexts(idx)
+ if (cached != null) {
+ cached
+ } else {
+ val text = value.asInstanceOf[UTF8String].toString
+ validateJsonText(idx, text)
+ cachedValidatedTexts(idx) = text
+ text
+ }
+ } else {
+ val text = value.asInstanceOf[UTF8String].toString
+ validateJsonText(idx, text)
+ text
+ }
+ }
+
+ // A foldable, non-null key (the common literal-key case, e.g.
`JSON_OBJECT('id' VALUE col)`)
+ // renders to the same JSON member name on every row, so render it once here
instead of re-running
+ // the Jackson writer per row. `renderedKeys(i)` holds that cached name, or
`null` when the key
+ // must be evaluated and rendered per row -- a non-foldable key, or a
foldable key that evaluates
+ // to `null` (still reported as JSON_OBJECT_NULL_KEY at eval).
+ @transient private lazy val renderedKeys: Array[String] = memberArray.map {
case (k, _) =>
+ if (k.foldable) {
+ val key = k.eval(EmptyRow)
+ if (key == null) null else renderKey(key)
+ } else {
+ null
+ }
+ }
+
+ override def eval(input: InternalRow): Any = {
+ val sb = new java.lang.StringBuilder("{")
+ var first = true
+ var i = 0
+ val localMembers = memberArray
+ val localRawJson = rawJsonArray
+ while (i < localMembers.length) {
+ val (keyExpr, valueExpr) = localMembers(i)
+ // Resolve the member's key. A foldable non-null key is already rendered
and cached. Otherwise
+ // evaluate the key before the value so a null key deterministically
raises
+ // JSON_OBJECT_NULL_KEY, independent of whether the value expression
happens to throw.
+ // Delay rendering dynamic keys until we know the member will be
emitted, avoiding work for
+ // null values under ABSENT ON NULL.
+ val cachedKey = renderedKeys(i)
+ val key = if (cachedKey == null) {
+ val key = keyExpr.eval(input)
+ if (key == null) {
+ throw QueryExecutionErrors.jsonObjectNullKeyError()
+ }
+ key
+ } else {
+ null
+ }
+ val value = valueExpr.eval(input)
+ if (value != null || nullBehavior == JsonConstructorNullBehavior.Null) {
+ if (!first) sb.append(",")
+ first = false
+ val keyName = if (cachedKey != null) cachedKey else renderKey(key)
+ sb.append(keyName).append(":")
+ if (value == null) {
+ sb.append("null")
+ } else if (localRawJson(i)) {
+ sb.append(rawJsonText(i, value))
+ } else {
+ appendRenderedValue(sb, i, value)
+ }
+ }
+ i += 1
+ }
+ sb.append("}")
+ val jsonStr = UTF8String.fromString(sb.toString)
+ if (returning == StringType) {
+ jsonStr
+ } else {
+ castInput.update(0, jsonStr)
+ returningCast.eval(castInput)
+ }
+ }
+
+ override def prettyName: String = "json_object"
+
+ override def sql: String = {
Review Comment:
Done.
##########
docs/sql-ref-syntax-qry-select-json-object.md:
##########
@@ -0,0 +1,168 @@
+---
+layout: global
+title: JSON_OBJECT
+displayTitle: JSON_OBJECT
+license: |
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+---
+
+### Description
+
+The `JSON_OBJECT` constructor function builds a JSON object from key/value
pairs and returns it as
+JSON text. This is the SQL-standard way (SQL:2016) to assemble a JSON object
inline, with common
+compatibility syntax from other systems. `JSON_OBJECT` is an expression that
can appear anywhere a
+value is allowed.
+
+Each value is serialized with the same JSON writer as the built-in `to_json`
function, so numbers,
+decimals, booleans, dates, timestamps, and nested structs/arrays/maps render
the same way.
+Null-field handling inside a struct value therefore follows
+`spark.sql.jsonGenerator.ignoreNullFields`, exactly as `to_json` does; the `ON
NULL` clause below
+controls only the top-level object members.
+
+This is an initial subset of the SQL:2016 `JSON_OBJECT` constructor. It
supports the key/value
+members, the value-level `FORMAT JSON` marker, the `{ NULL | ABSENT } ON NULL`
clause, and a
+string-type `RETURNING`. The following SQL/JSON clauses are not yet supported:
+
+* The `{ WITH | WITHOUT } UNIQUE KEYS` clause. Duplicate keys are kept in
source order (the
+ `WITHOUT UNIQUE KEYS` behavior); there is no option to reject them.
+* `RETURNING` to a non-string type, or with a binary/`FORMAT JSON` output
clause. The result is
+ always JSON text of a string type.
+
+### Syntax
+
+```sql
+JSON_OBJECT ( [ { key VALUE value | KEY key VALUE value | key : value } [
FORMAT JSON ] [, ...] ]
+ [ { NULL | ABSENT } ON NULL ]
+ [ RETURNING data_type ] )
+
+JSON_OBJECT ( [ key, value [, key, value] ... ]
+ [ { NULL | ABSENT } ON NULL ]
+ [ RETURNING data_type ] )
+```
+
+### Parameters
+
+* **key**
+
+ An expression producing a member name. It must be a non-null string; a
`NULL` key raises an
+ error at runtime. The key is validated before the `ON NULL` handling, so a
`NULL` key raises the
+ error even when the member's value is `NULL` and `ABSENT ON NULL` would
otherwise omit it.
+
+* **value**
+
+ An expression producing the member value. Values may have different types
and may be nested
+ JSON constructors. `JSON_OBJECT()` with no members produces the empty
object `{}`. A value is
+ spliced in as raw JSON, rather than quoted as a string, only when it
carries `FORMAT JSON` (see
+ below) -- either the explicit clause or the implicit form carried by a
recognized JSON producer
+ written directly in the value position. This covers a nested `JSON_OBJECT`
or `JSON_ARRAY` (e.g.
+ `JSON_OBJECT('a' VALUE JSON_OBJECT('b' VALUE 1))` produces
`{"a":{"b":1}}`), and a nested
+ `JSON_QUERY` under the default `KEEP QUOTES`, which returns JSON text (e.g.
+ `JSON_OBJECT('a' VALUE JSON_QUERY('{"o":{"x":1}}', '$.o'))` produces
`{"a":{"x":1}}`). A
+ `JSON_QUERY` with `OMIT QUOTES` returns an ordinary string, so it is
quoted like any other value
+ (e.g. `JSON_OBJECT('n' VALUE JSON_QUERY('{"n":"Ada"}', '$.n' OMIT
QUOTES))` produces
+ `{"n":"Ada"}`). Without `FORMAT JSON`, an ordinary string value is always
quoted, even when its
+ contents are themselves valid JSON text (e.g. `JSON_OBJECT('a' VALUE
'{"b":1}')` produces
+ `{"a":"{\"b\":1}"}`). Implicit raw splicing applies only to the direct
syntax above: a qualified
+ or otherwise routed call (e.g. `builtin.json_object('a', json_object('b',
1))`) currently quotes
Review Comment:
Done.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -2272,3 +2326,409 @@ case class JsonTypeof(child: Expression)
override protected def withNewChildInternal(newChild: Expression):
JsonTypeof =
copy(child = newChild)
}
+
+/**
+ * The SQL:2016 `JSON_OBJECT` constructor function (feature T811): constructs
a JSON object from
+ * key-value pairs, written `key VALUE value`, `KEY key VALUE value`, `key :
value`, or the
+ * MySQL-style `key, value`.
+ *
+ * Keys must be non-null strings; a null key is an error. The `ON NULL` clause
controls whether
+ * null-valued pairs are included (NULL ON NULL, the standard default) or
omitted (ABSENT ON NULL).
+ *
+ * A value is spliced in raw (unquoted) when it carries `FORMAT JSON` (tracked
per member by
+ * `rawJson`): the explicit clause (e.g. `JSON_OBJECT('a' VALUE '{"b":1}'
FORMAT JSON)`), or the
+ * implicit form carried by a nested JSON constructor / `JSON_QUERY` under
KEEP QUOTES (see
+ * `rawJsonValue`). Explicit `FORMAT JSON` on a non-constructor is arbitrary
user text, so
+ * `needsValidation` marks it for JSON validation at eval; a nested
constructor is trusted.
+ *
+ * Examples:
+ * JSON_OBJECT('id' VALUE 7, 'name' VALUE 'Ada') ->
'{"id":7,"name":"Ada"}'
+ * JSON_OBJECT('id': 7, 'v': NULL) -> '{"id":7,"v":null}'
+ * JSON_OBJECT('id': 7, 'v': NULL ABSENT ON NULL) -> '{"id":7}'
+ * JSON_OBJECT() -> '{}'
+ *
+ * A clause-free call that is not itself lexically nested in an enclosing
constructor's *value*
+ * position routes through function resolution and is rebuilt by
`JsonObjectExpressionBuilder`, so a
+ * same-named routine can shadow the built-in `json_object`: always for the
comma form (a nested
+ * producer value routes too, carrying its raw-splice eligibility via
`JsonImplicitFormatCarrier`),
+ * but for the `VALUE` / `:` forms only when no member carries a raw value
(`rawJson.contains(true)`
+ * -- an explicit `FORMAT JSON` or a nested producer in *value* position --
keeps it direct).
+ * Otherwise it is built directly from the grammar (clause-bearing, or
value-nested); a nested
+ * *key*-position call stays routable (see `AstBuilder.visitJsonObject`).
+ * The user-facing reference lives in
`docs/sql-ref-syntax-qry-select-json-object.md`.
+ */
+case class JsonObjectExpr(
+ members: Seq[(Expression, Expression)],
+ rawJson: Seq[Boolean],
+ needsValidation: Seq[Boolean],
+ nullBehavior: JsonConstructorNullBehavior =
JsonConstructorNullBehavior.Null,
+ returning: DataType = StringType,
+ timeZoneId: Option[String] = None)
+ extends Expression
+ with TimeZoneAwareExpression
+ with CodegenFallback
+ with QueryErrorsBase
+ // Default RETURNING is a plain STRING, so
`DefaultStringProducingExpression` lets
+ // `ApplyDefaultCollation` cast the result to a non-default collation; the
`dataType` override
+ // below stays authoritative when RETURNING is given explicitly.
+ with DefaultStringProducingExpression
+ with ImplicitlyFormattedAsJson {
+
+ // `rawJson(i)` splices member `i`'s value verbatim (already-JSON text)
rather than quoting it;
+ // `needsValidation(i)` marks that raw text as arbitrary user input to
JSON-validate at eval (an
+ // explicit `FORMAT JSON` on a non-constructor). Both are frozen at parse
time (see
+ // `AstBuilder.visitJsonObject`). A validated value is always a spliced one.
+ require(members.length == rawJson.length && members.length ==
needsValidation.length,
+ "JsonObjectExpr requires one rawJson and one needsValidation flag per
member")
+ require(needsValidation.lazyZip(rawJson).forall((nv, raw) => !nv || raw),
+ "JsonObjectExpr needsValidation implies rawJson")
+
+ @transient private lazy val memberArray: Array[(Expression, Expression)] =
+ members.toArray
+
+ @transient private lazy val rawJsonArray: Array[Boolean] =
+ rawJson.toArray
+
+ @transient private lazy val needsValidationArray: Array[Boolean] =
+ needsValidation.toArray
+
+ // Always throwable: a null key raises JSON_OBJECT_NULL_KEY at eval even
with non-throwable
+ // children, which keeps the optimizer from pushing it below a filtering
join. Left non-foldable
+ // (the default) so folding a constant call does not eagerly raise that
null-key error at
+ // optimization for rows a filter would later drop.
+ override lazy val throwable: Boolean = true
+
+ // The value is never null, but report nullable as `throwable` (like
JsonArray): `NullPropagation`
+ // keys off `nullable`, so a non-nullable constructor would let it fold `IS
[NOT] NULL` to a
+ // constant or `count(...)` to `count(1)`, skipping the eval that must raise
JSON_OBJECT_NULL_KEY.
+ override def nullable: Boolean = throwable
+
+ override def dataType: DataType = returning
+
+ // Flattened `[k0, v0, k1, v1, ...]` view, cached since `children` is walked
repeatedly.
+ @transient private lazy val childrenSeq: Seq[Expression] =
+ memberArray.flatMap { case (k, v) => Seq(k, v) }.toImmutableArraySeq
+
+ override def children: Seq[Expression] = childrenSeq
+
+ override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+ copy(timeZoneId = Option(timeZoneId))
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ val inputCheck = super.checkInputDataTypes()
+ if (inputCheck.isFailure) {
+ inputCheck
+ } else if (!JsonObjectExpr.isValidReturningType(returning)) {
+ DataTypeMismatch(
+ errorSubClass = "INVALID_JSON_RETURNING_TYPE",
+ messageParameters = Map(
+ "functionName" -> toSQLId(prettyName),
+ "returningType" -> toSQLType(returning)))
+ } else {
+ // Keys must be character strings (they become JSON object member
names). Reject any other
+ // concrete type up front so a non-string key cannot reach the
StringType Jackson writer and
+ // fail with an internal ClassCastException. NullType is allowed
through: a null key value is
+ // reported at runtime as JSON_OBJECT_NULL_KEY rather than a type error.
+ val keyCheck = memberArray.iterator.map(_._1).zipWithIndex.collectFirst {
+ case (k, keyIndex) if !(k.dataType.isInstanceOf[StringType] ||
k.dataType == NullType) =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> ordinalNumber(keyIndex * 2),
+ "requiredType" -> toSQLType(StringType),
+ "inputSql" -> toSQLExpr(k),
+ "inputType" -> toSQLType(k.dataType)))
+ }.getOrElse(TypeCheckResult.TypeCheckSuccess)
+ if (keyCheck.isFailure) {
+ keyCheck
+ } else {
+ // Every value must be serializable to JSON. This mirrors `to_json`'s
analysis-time
+ // `JacksonUtils.verifyType` check, plus a guard for the spatial
atomics that check accepts
+ // but `JacksonGenerator` cannot write (rejecting a runtime failure at
analysis instead).
+ memberArray.iterator.map(_._2).zipWithIndex
+ .foldLeft(TypeCheckResult.TypeCheckSuccess: TypeCheckResult) {
+ case (acc, _) if acc.isFailure => acc
+ case (_, (v, valueIndex)) if rawJsonArray(valueIndex) &&
+ !(v.dataType.isInstanceOf[StringType] || v.dataType ==
NullType) =>
+ // A raw-spliced value must carry JSON text (string); NullType
is allowed (eval drops
+ // or emits it before splicing). Reject any other type up front,
matching JSON_ARRAY's
+ // FORMAT JSON check, rather than failing with a
ClassCastException at eval.
+ DataTypeMismatch(
+ errorSubClass = "INVALID_JSON_FORMAT_JSON_INPUT",
+ messageParameters = Map(
+ "functionName" -> toSQLId(prettyName),
+ // Flattened 1-based arg position, not the member ordinal
(value of member i is
+ // arg 2i+2), matching the key check's
`ordinalNumber(keyIndex*2)`.
+ "position" -> (valueIndex * 2 + 2).toString,
+ "inputType" -> toSQLType(v.dataType)))
+ case (_, (v, _)) =>
+ val elemCheck = JacksonUtils.verifyType(prettyName, v.dataType)
+ if (elemCheck.isFailure) {
+ elemCheck
+ } else if
(JsonObjectExpr.containsUnsupportedJsonType(v.dataType)) {
+ DataTypeMismatch(
+ errorSubClass = "CANNOT_CONVERT_TO_JSON",
+ messageParameters = Map(
+ "name" -> toSQLId(prettyName),
+ "type" -> toSQLType(v.dataType)))
+ } else {
+ TypeCheckResult.TypeCheckSuccess
+ }
+ }
+ }
+ }
+ }
+
+ override def stateful: Boolean = true
+
+ @transient private lazy val resolvedZoneId: String =
+ timeZoneId.getOrElse(SQLConf.get.sessionLocalTimeZone)
+
+ // JacksonGenerator (shared with `to_json`, so rendering matches) can only
serialize a container,
+ // not a bare scalar, so each key/value is wrapped in a one-element array
whose brackets are
+ // stripped in renderKey/appendRenderedValue below.
+ @transient private lazy val keyEvaluator: StructsToJsonEvaluator =
+ StructsToJsonEvaluator(Map.empty, ArrayType(StringType),
Some(resolvedZoneId))
+
+ @transient private lazy val valueEvaluators: Array[StructsToJsonEvaluator] =
+ memberArray.map { case (_, v) =>
+ StructsToJsonEvaluator(Map.empty, ArrayType(v.dataType),
+ Some(resolvedZoneId))
+ }
+
+ @transient private lazy val singleElem: Array[Any] = new Array[Any](1)
+
+ @transient private lazy val singleElemData: GenericArrayData =
+ new GenericArrayData(singleElem)
+
+ @transient private lazy val castInput: GenericInternalRow =
+ new GenericInternalRow(1)
+
+ @transient private lazy val returningCast: Expression =
+ Cast(BoundReference(0, StringType, nullable = true), returning, timeZoneId,
+ EvalMode.ANSI)
+
+ // `evaluateString` returns the Java String directly, skipping `evaluate`'s
UTF8String round trip.
+ // The key is cached or appended whole, so it still materializes the
interior substring.
+ private def renderKey(key: Any): String = {
+ singleElem(0) = key
+ val arrJson = keyEvaluator.evaluateString(singleElemData)
+ arrJson.substring(1, arrJson.length - 1)
+ }
+
+ // Append the interior of the "[<frag>]" wrapper straight into the builder
(no per-value
+ // substring), mirroring `JsonArray.appendRenderedElement`.
+ private def appendRenderedValue(sb: java.lang.StringBuilder, idx: Int,
value: Any): Unit = {
+ singleElem(0) = value
+ val arrJson = valueEvaluators(idx).evaluateString(singleElemData)
+ sb.append(arrJson, 1, arrJson.length - 1)
+ }
+
+ @transient private lazy val formatJsonFactory: JsonFactory = new
JsonFactory()
+
+ @transient private lazy val cachedValidatedTexts: Array[String] =
+ new Array[String](memberArray.length)
+
+ // A `FORMAT JSON` value is spliced verbatim, so it must be exactly one
well-formed JSON value
+ // (a string like `1,2` or `{bad` would corrupt the object). Parse one
value; reject anything
+ // trailing. Mirrors `JsonArray.validateJsonText`.
+ private def validateJsonText(idx: Int, text: String): Unit = {
+ val valid =
+ try {
+ Utils.tryWithResource(formatJsonFactory.createParser(text)) { parser =>
+ if (parser.nextToken() == null) {
+ false
+ } else {
+ parser.skipChildren()
+ parser.nextToken() == null
+ }
+ }
+ } catch {
+ case _: JsonProcessingException => false
+ }
+ if (!valid) {
+ // Flattened 1-based value-arg position (member `idx` is arg `2*idx+2`),
matching the
+ // analysis-time INVALID_JSON_FORMAT_JSON_INPUT check.
+ throw QueryExecutionErrors.invalidJsonFormatJsonValueError(prettyName,
idx * 2 + 2, text)
+ }
+ }
+
+ // Raw text to splice for member `idx`: a trusted nested constructor splices
as-is; an explicit
+ // `FORMAT JSON` string is validated once (cached) when foldable, else per
row.
+ private def rawJsonText(idx: Int, value: Any): String = {
+ if (!needsValidationArray(idx)) {
+ value.asInstanceOf[UTF8String].toString
+ } else if (memberArray(idx)._2.foldable) {
+ val cached = cachedValidatedTexts(idx)
+ if (cached != null) {
+ cached
+ } else {
+ val text = value.asInstanceOf[UTF8String].toString
+ validateJsonText(idx, text)
+ cachedValidatedTexts(idx) = text
+ text
+ }
+ } else {
+ val text = value.asInstanceOf[UTF8String].toString
+ validateJsonText(idx, text)
+ text
+ }
+ }
+
+ // A foldable, non-null key (the common literal-key case, e.g.
`JSON_OBJECT('id' VALUE col)`)
+ // renders to the same JSON member name on every row, so render it once here
instead of re-running
+ // the Jackson writer per row. `renderedKeys(i)` holds that cached name, or
`null` when the key
+ // must be evaluated and rendered per row -- a non-foldable key, or a
foldable key that evaluates
+ // to `null` (still reported as JSON_OBJECT_NULL_KEY at eval).
+ @transient private lazy val renderedKeys: Array[String] = memberArray.map {
case (k, _) =>
+ if (k.foldable) {
+ val key = k.eval(EmptyRow)
+ if (key == null) null else renderKey(key)
+ } else {
+ null
+ }
+ }
+
+ override def eval(input: InternalRow): Any = {
+ val sb = new java.lang.StringBuilder("{")
+ var first = true
+ var i = 0
+ val localMembers = memberArray
+ val localRawJson = rawJsonArray
+ while (i < localMembers.length) {
+ val (keyExpr, valueExpr) = localMembers(i)
+ // Resolve the member's key. A foldable non-null key is already rendered
and cached. Otherwise
+ // evaluate the key before the value so a null key deterministically
raises
+ // JSON_OBJECT_NULL_KEY, independent of whether the value expression
happens to throw.
+ // Delay rendering dynamic keys until we know the member will be
emitted, avoiding work for
+ // null values under ABSENT ON NULL.
+ val cachedKey = renderedKeys(i)
+ val key = if (cachedKey == null) {
+ val key = keyExpr.eval(input)
+ if (key == null) {
+ throw QueryExecutionErrors.jsonObjectNullKeyError()
+ }
+ key
+ } else {
+ null
+ }
+ val value = valueExpr.eval(input)
+ if (value != null || nullBehavior == JsonConstructorNullBehavior.Null) {
+ if (!first) sb.append(",")
+ first = false
+ val keyName = if (cachedKey != null) cachedKey else renderKey(key)
+ sb.append(keyName).append(":")
+ if (value == null) {
+ sb.append("null")
+ } else if (localRawJson(i)) {
+ sb.append(rawJsonText(i, value))
+ } else {
+ appendRenderedValue(sb, i, value)
+ }
+ }
+ i += 1
+ }
+ sb.append("}")
+ val jsonStr = UTF8String.fromString(sb.toString)
+ if (returning == StringType) {
+ jsonStr
+ } else {
+ castInput.update(0, jsonStr)
+ returningCast.eval(castInput)
+ }
+ }
+
+ override def prettyName: String = "json_object"
+
+ override def sql: String = {
+ val membersSQL = members.zip(rawJson).map { case ((k, v), raw) =>
+ // Render SQL that reparses to the frozen `raw` flag, regardless of
optimizer rewrites of the
+ // child:
+ // - raw + child still a bare implicit producer (bare, or under a
pass-through COLLATE):
+ // render that bare constructor so reparse re-derives implicit raw
splicing.
+ // - raw + child no longer recoverable as a constructor
(constant-folded to a literal, or a
+ // Cast wrapper): render an explicit `FORMAT JSON` so reparse splices
it raw.
+ // - quoted + child would reparse as implicit JSON (an
optimizer-inlined constructor, bare or
+ // under COLLATE): neutralize with `CAST(... AS STRING)` so reparse
keeps it quoted.
+ // - otherwise the child's shape already reproduces the flag.
+ val valueSQL = (raw, JsonObjectExpr.rawJsonValue(v)) match {
+ case (true, Some(c)) => c.sql
+ case (true, None) => s"${v.sql} FORMAT JSON"
+ case (false, Some(_)) => s"CAST(${v.sql} AS STRING)"
+ case (false, None) => v.sql
+ }
+ s"${k.sql} VALUE $valueSQL"
+ }.mkString(", ")
+ // Always render an explicit ON NULL clause. It forces the direct grammar
path on reparse (a
+ // clause-free call routes through function resolution and could bind a
shadowing routine), so a
+ // resolved built-in round-trips to the same built-in. This diverges from
`JsonArray`, which
+ // omits its default clause and has the same latent gap.
+ // TODO(SPARK-59728): align `JsonArray.sql` to render an explicit
direct-path marker too.
+ val nullSQL = nullBehavior match {
+ case JsonConstructorNullBehavior.Null => " NULL ON NULL"
+ case JsonConstructorNullBehavior.Absent => " ABSENT ON NULL"
Review Comment:
Done.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -1745,6 +1746,59 @@ object JsonArrayExpressionBuilder extends
ExpressionBuilder {
}
}
+/**
+ * Marks a routed comma-form `JSON_OBJECT(...)` value as a lexically nested
JSON producer (implicit
+ * `FORMAT JSON`). Transparent (keeps its child's type, value, foldability),
so it does not perturb
+ * a shadowing routine's overload selection; the unshadowed built-in unwraps
it and splices raw.
+ * Attached only by direct `JSON_OBJECT(...)` syntax, so qualified/generic
calls keep quoting
+ * pending SPARK-59243.
+ */
+case class JsonImplicitFormatCarrier(child: Expression) extends
TaggingExpression {
+ // Render as the bare child so generated SQL for a carrier that survives
into a routine's plan
+ // stays reparseable (the built-in unwraps it before rendering its own SQL).
+ override def sql: String = child.sql
+ override protected def withNewChildInternal(newChild: Expression):
JsonImplicitFormatCarrier =
+ copy(child = newChild)
+}
+
+@ExpressionDescription(
+ usage =
+ "_FUNC_([key, value [, key, value ...]]) - Returns a JSON object string
built from the " +
+ "given key/value pairs.",
+ arguments = """
+ Arguments:
+ * key - a non-null string naming an object member.
+ * value - the member's value.
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_('id', 7, 'name', 'Ada');
+ {"id":7,"name":"Ada"}
+ > SELECT _FUNC_();
+ {}
+ """,
+ since = "4.4.0",
+ group = "json_funcs")
+object JsonObjectExpressionBuilder extends ExpressionBuilder {
+ override def build(funcName: String, expressions: Seq[Expression]):
Expression = {
+ // The routed form is a flat `key, value, ...` list (the MySQL-style
spelling), so it must have
+ // an even length; an odd count leaves a dangling key with no value.
+ if (expressions.length % 2 != 0) {
+ throw QueryCompilationErrors.wrongNumArgsError(
+ funcName, Seq("2n (n >= 0)"), expressions.length)
+ }
+ // A `JsonImplicitFormatCarrier` marks a lexically nested JSON producer,
spliced raw (unwrapped
+ // here) and trusted; every other value is quoted. Routed calls carry no
explicit FORMAT JSON,
+ // so nothing needs validation.
+ val taggedMembers = expressions.grouped(2).map {
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
