spark-test commented on code in PR #58024:
URL: https://github.com/apache/spark/pull/58024#discussion_r4472316912
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala:
##########
@@ -1138,4 +1139,144 @@ class JsonExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
StructType(StructField("f", FileType.external) :: Nil))
}
}
+
+ test("SPARK-58793: IS JSON evaluates its operand exactly once") {
+ // Regression test: a non-deterministic operand (e.g. `IF(rand() < 0.5,
NULL, '{}')`) must be
+ // evaluated a single time, otherwise the predicate could observe two
different values and
+ // return a wrong result (false, when only NULL or true are valid). The
replacement must
+ // therefore reference the operand exactly once.
+ val child = Literal.create("{}", StringType)
+ Seq(IsJsonShape.Any, IsJsonShape.Value, IsJsonShape.Object,
IsJsonShape.Array,
+ IsJsonShape.Scalar).foreach { shape =>
+ val references = IsJson(child, shape).replacement.collect { case e if e
eq child => e }
+ assert(references.length == 1,
+ s"operand referenced ${references.length} times for shape $shape,
expected exactly once")
+ }
+ }
+
+ test("SPARK-58793: IS JSON requires strict ANSI JSON (no Hive leniency)") {
+ // Single quotes and unescaped control characters are allowed by the
Hive-compatible parser
+ // shared by json_typeof etc., but the ANSI SQL IS JSON predicate must
reject them.
+ checkEvaluation(IsJson(Literal("{'a':1}"), IsJsonShape.Any), false)
+ checkEvaluation(IsJson(Literal("['a', 'b']"), IsJsonShape.Any), false)
+ // A raw (unescaped) control character inside a JSON string is not valid
ANSI JSON.
+ checkEvaluation(IsJson(Literal("\"line1\nline2\""), IsJsonShape.Any),
false)
+ // The double-quoted equivalents are well-formed ANSI JSON.
+ checkEvaluation(IsJson(Literal("""{"a":1}"""), IsJsonShape.Any), true)
+ checkEvaluation(IsJson(Literal("""["a", "b"]"""), IsJsonShape.Any), true)
+ // Other parser extensions are rejected too: comments, unquoted field
names, non-numeric
+ // numbers, leading zeros, and whitespace-only input.
+ Seq("/* c */ {}", "{} // c", "# c\n{}", "{a:1}", "NaN", "Infinity",
"-Infinity", "01",
+ "-01", "[01]", " ", "\n\t").foreach { json =>
+ checkEvaluation(IsJson(Literal(json), IsJsonShape.Any), false)
+ }
+ // Duplicate field names are accepted (no WITH UNIQUE KEYS support).
+ checkEvaluation(IsJson(Literal("""{"a":1,"a":2}"""), IsJsonShape.Object),
true)
+ }
+
+ test("SPARK-58793: IS JSON rejects structurally malformed containers") {
+ // Containers that only turn malformed after a valid first token:
truncated input, trailing
+ // commas, missing values, a missing colon, and mismatched or unmatched
closers. Pinning these
+ // guards against a strict factory accidentally built with
JsonReadFeature.ALLOW_TRAILING_COMMA
+ // or ALLOW_MISSING_VALUES, or against a token walk that accepts a
container ending early.
+ Seq("{", "[", "[1,", "{\"a\"", "{\"a\":", "[1,]", "{\"a\":1,}", "[1,,2]",
"{\"a\"}", "[1}",
+ "{]", "}", "]").foreach { json =>
+ checkEvaluation(IsJson(Literal(json), IsJsonShape.Any), false)
+ }
+ // The matching shape qualifier must reject them too, not only bare IS
JSON: the first token
+ // matches the shape, so only the malformed rest can reject the input.
+ checkEvaluation(IsJson(Literal("[1,]"), IsJsonShape.Array), false)
+ checkEvaluation(IsJson(Literal("{\"a\":1,}"), IsJsonShape.Object), false)
+ checkEvaluation(IsJson(Literal("[1,,2]"), IsJsonShape.Array), false)
+ checkEvaluation(IsJson(Literal("[1}"), IsJsonShape.Array), false)
+ }
+
+ test("SPARK-58793: IS JSON fails on input beyond the parser limits, like
parse_json") {
+ def checkLimit(
+ json: String,
+ shape: IsJsonShape,
+ condition: String,
+ parameters: Map[String, String]): Unit = {
+ // Only the first 128 characters of the input are echoed in the error.
+ checkErrorInExpression[SparkRuntimeException](
+ IsJson(Literal(json), shape),
+ InternalRow.empty,
+ condition,
+ parameters + ("inputPrefix" -> s"'${json.take(128)}...'"))
+ }
+ // Inputs at the limits are classified as usual.
+ checkEvaluation(IsJson(Literal("[" * 1000 + "]" * 1000),
IsJsonShape.Array), true)
+ checkEvaluation(IsJson(Literal("1" * 1000), IsJsonShape.Scalar), true)
+ // One past the nesting depth, number length, or field name length limit
is an error rather
Review Comment:
**Nit (P3):** This block covers three of the four documented limits. Nothing
reaches `IS_JSON_INPUT_LIMIT_EXCEEDED.STRING_LENGTH`. That branch rests on two
premises no test checks. First, `limitError` must recognize Jackson's "String
value length" message. Second, the token walk must keep decoding every string
value through `getText()`, which is what makes Jackson enforce the limit.
If a Jackson upgrade rewords the message, the error silently becomes
`UNKNOWN`. If the walk stops decoding strings, an over-long string returns
true. A case with a string value of
`StreamReadConstraints.DEFAULT_MAX_STRING_LEN + 1` characters, as
`JsonFunctionsSuite` already builds, would pin both.
**Verification:**
- **Regression:** The new case fails if the string-length message stops
being recognized, which would yield the UNKNOWN sub-condition, or if string
values stop being decoded during the walk, which would yield true.
##########
sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionUtils.java:
##########
@@ -117,4 +122,151 @@ public static UTF8String jsonTypeof(UTF8String json) {
return null;
}
}
+
+ // Shape codes for the SQL `IS [NOT] JSON` predicate, kept in sync with
IsJsonShape on the
+ // Scala side. SHAPE_ANY covers both bare `IS JSON` and `IS JSON VALUE` (any
well-formed value).
+ public static final int SHAPE_ANY = 0;
+ public static final int SHAPE_OBJECT = 1;
+ public static final int SHAPE_ARRAY = 2;
+ public static final int SHAPE_SCALAR = 3;
+
+ // A strict JSON factory for the ANSI SQL `IS [NOT] JSON` predicate. Unlike
the shared factory
+ // used by the Hive-compatible JSON functions, it leaves ALLOW_SINGLE_QUOTES
and
+ // ALLOW_UNESCAPED_CONTROL_CHARS disabled, so only well-formed ANSI JSON is
accepted
+ // (e.g. `{'a':1}` with single quotes is rejected).
+ private static final JsonFactory STRICT_JSON_FACTORY = new JsonFactory();
+
+ /**
+ * Evaluates the SQL `IS JSON` predicate for a single, already-evaluated
operand: returns whether
+ * {@code json} is a well-formed ANSI JSON value of the requested {@code
shape}. The operand is
+ * read exactly once here, so a non-deterministic input is evaluated a
single time (unlike a
+ * plan that references the child in more than one place). NULL handling
(NULL input yields a
+ * NULL result) is left to the calling expression; this method is only
invoked for non-null
+ * input. Malformed input yields false. Input that exceeds one of Jackson's
default
+ * {@code StreamReadConstraints} (nesting depth, number length, string
length, or field name
+ * length) cannot be classified and raises an IS_JSON_INPUT_LIMIT_EXCEEDED
condition, at the
+ * same limits where {@code parse_json} fails -- but only if checking the
input actually
+ * requires going past a limit: a shape mismatch decided by the first token
(including an
+ * over-long root number under OBJECT or ARRAY, which cannot be an object or
an array)
+ * returns false, like any other mismatch.
+ */
+ public static boolean isJson(UTF8String json, int shape) {
+ // The parser decodes the bytes through a UTF-8 reader that silently
replaces malformed
+ // sequences with U+FFFD, so check the raw bytes first: input that is not
valid UTF-8
+ // (e.g. a lone continuation byte, or an encoded surrogate) is not
well-formed JSON text.
+ if (!json.isValid()) {
+ return false;
+ }
+ try (JsonParser jsonParser =
+ CreateJacksonParser.utf8String(STRICT_JSON_FACTORY, json)) {
+ JsonToken token;
+ try {
+ token = jsonParser.nextToken();
+ } catch (StreamConstraintsException e) {
+ // Jackson validates a number's length while producing its token, and
no other limit can
+ // be reached while producing the first token (nesting needs a second
token, and strings
Review Comment:
**Nit (P3):** The parenthetical is half right. Jackson defers decoding
string values until `getText()`, but it decodes a field name when `nextToken()`
produces the `FIELD_NAME` token, and the name is available from `currentName()`
without `getText()`. The conclusion still holds because the first token can
never be a field name, and the comment could give that reason instead: nesting
needs a second token, string values are decoded lazily, and a first token is
never a field name. As written, it misleads anyone who extends this handler or
the trailing-content check about which limits can fire where.
**Verification:**
- **Inspection:** The first-token handler's comment no longer claims that
field names are decoded only at getText, and it justifies the over-long-number
premise from the kinds of token that can come first.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala:
##########
@@ -2689,3 +2689,70 @@ object JsonObjectExpr {
case _ => false
}
}
+
+// Shape qualifier for IS [NOT] JSON: which top-level JSON type to require.
+sealed trait IsJsonShape
+object IsJsonShape {
+ // Per ANSI SQL:2016 T821, IS JSON VALUE is semantically equivalent to bare
IS JSON:
+ // both accept any well-formed JSON value (object, array, or scalar). They
are kept as
+ // separate shapes so the sql() output round-trips faithfully.
+ case object Any extends IsJsonShape // IS JSON (no qualifier)
+ case object Value extends IsJsonShape // IS JSON VALUE
+ case object Scalar extends IsJsonShape // IS JSON SCALAR -- primitives only
(not object/array)
+ case object Array extends IsJsonShape // IS JSON ARRAY
+ case object Object extends IsJsonShape // IS JSON OBJECT
+}
+
+case class IsJson(child: Expression, shape: IsJsonShape = IsJsonShape.Any)
Review Comment:
**Nit (P3):** `IsJson` is built by the parser but is neither a `Predicate`
nor listed in `ResolverGuard.isGenerallySupportedExpression`, so the
single-pass analyzer's guard rejects every `IS [NOT] JSON` query, and `NOT (...
IS JSON)` through its child. Its parser-built scalar siblings `JsonArray`,
`JsonObjectExpr`, `JsonValue`, `JsonQuery` and `JsonExists` are all in the
guard's JSON arm and in ResolverGuardSuite's "SQL JSON constructor and path
functions" test. `JSON_OBJECT` added its entry in the same change that
introduced it.
As a result, IS JSON gets no single-pass resolution and no dual-run
comparison in tests, where
`spark.sql.analyzer.singlePassResolver.dualRunWithLegacy` defaults to true.
Results under the default configuration are unaffected, because the single-pass
resolver is off. Adding `IsJson` to that JSON arm, with guard-suite cases,
would close the gap.
**Verification:**
- **Behavior:** The resolver guard accepts SELECT '{}' IS JSON, SELECT '{}'
IS NOT JSON OBJECT and SELECT NULL IS JSON.
- **Compatibility:** With dual-run analysis enabled, IS JSON queries over
STRING, collated, CHAR/VARCHAR and untyped NULL operands produce identical
plans from both analyzers.
##########
sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/json/JsonExpressionUtils.java:
##########
@@ -117,4 +122,151 @@ public static UTF8String jsonTypeof(UTF8String json) {
return null;
}
}
+
+ // Shape codes for the SQL `IS [NOT] JSON` predicate, kept in sync with
IsJsonShape on the
+ // Scala side. SHAPE_ANY covers both bare `IS JSON` and `IS JSON VALUE` (any
well-formed value).
+ public static final int SHAPE_ANY = 0;
+ public static final int SHAPE_OBJECT = 1;
+ public static final int SHAPE_ARRAY = 2;
+ public static final int SHAPE_SCALAR = 3;
+
+ // A strict JSON factory for the ANSI SQL `IS [NOT] JSON` predicate. Unlike
the shared factory
+ // used by the Hive-compatible JSON functions, it leaves ALLOW_SINGLE_QUOTES
and
+ // ALLOW_UNESCAPED_CONTROL_CHARS disabled, so only well-formed ANSI JSON is
accepted
+ // (e.g. `{'a':1}` with single quotes is rejected).
+ private static final JsonFactory STRICT_JSON_FACTORY = new JsonFactory();
+
+ /**
+ * Evaluates the SQL `IS JSON` predicate for a single, already-evaluated
operand: returns whether
+ * {@code json} is a well-formed ANSI JSON value of the requested {@code
shape}. The operand is
+ * read exactly once here, so a non-deterministic input is evaluated a
single time (unlike a
+ * plan that references the child in more than one place). NULL handling
(NULL input yields a
+ * NULL result) is left to the calling expression; this method is only
invoked for non-null
+ * input. Malformed input yields false. Input that exceeds one of Jackson's
default
+ * {@code StreamReadConstraints} (nesting depth, number length, string
length, or field name
+ * length) cannot be classified and raises an IS_JSON_INPUT_LIMIT_EXCEEDED
condition, at the
+ * same limits where {@code parse_json} fails -- but only if checking the
input actually
+ * requires going past a limit: a shape mismatch decided by the first token
(including an
+ * over-long root number under OBJECT or ARRAY, which cannot be an object or
an array)
+ * returns false, like any other mismatch.
+ */
+ public static boolean isJson(UTF8String json, int shape) {
+ // The parser decodes the bytes through a UTF-8 reader that silently
replaces malformed
+ // sequences with U+FFFD, so check the raw bytes first: input that is not
valid UTF-8
+ // (e.g. a lone continuation byte, or an encoded surrogate) is not
well-formed JSON text.
+ if (!json.isValid()) {
+ return false;
+ }
+ try (JsonParser jsonParser =
+ CreateJacksonParser.utf8String(STRICT_JSON_FACTORY, json)) {
+ JsonToken token;
+ try {
+ token = jsonParser.nextToken();
+ } catch (StreamConstraintsException e) {
+ // Jackson validates a number's length while producing its token, and
no other limit can
+ // be reached while producing the first token (nesting needs a second
token, and strings
+ // and field names are not decoded until getText), so the first value
is an over-long
+ // number. It can never be an object or an array, so the shape is
decided without
+ // classifying the number; the other shapes accept a number and still
need it classified.
+ if (shape == SHAPE_OBJECT || shape == SHAPE_ARRAY) {
+ return false;
+ }
+ throw limitError(e, json);
+ }
+ if (token == null) {
+ // Empty input is not a JSON value.
+ return false;
+ }
+ boolean matchesShape = switch (shape) {
+ case SHAPE_OBJECT -> token == JsonToken.START_OBJECT;
+ case SHAPE_ARRAY -> token == JsonToken.START_ARRAY;
+ case SHAPE_SCALAR -> switch (token) {
+ case VALUE_STRING, VALUE_NUMBER_INT, VALUE_NUMBER_FLOAT,
+ VALUE_TRUE, VALUE_FALSE, VALUE_NULL -> true;
+ default -> false;
+ };
+ // SHAPE_ANY: any well-formed JSON value (object, array, or scalar).
+ default -> switch (token) {
+ case START_OBJECT, START_ARRAY, VALUE_STRING, VALUE_NUMBER_INT,
+ VALUE_NUMBER_FLOAT, VALUE_TRUE, VALUE_FALSE, VALUE_NULL -> true;
+ default -> false;
+ };
+ };
+ if (!matchesShape) {
+ return false;
+ }
+ // Walk the whole value token by token (rather than `skipChildren`,
which does not look at
+ // string contents) so that malformed input (e.g. `{`, `[1,`) surfaces
as a parse error
+ // and every string value and field name can be checked for unpaired
surrogates.
+ int depth = 0;
+ do {
+ switch (token) {
+ case START_OBJECT, START_ARRAY -> depth++;
+ case END_OBJECT, END_ARRAY -> depth--;
+ case FIELD_NAME, VALUE_STRING -> {
+ if (hasUnpairedSurrogate(jsonParser.getText())) {
+ return false;
+ }
+ }
+ default -> { }
+ }
+ } while (depth > 0 && (token = jsonParser.nextToken()) != null);
+ if (depth > 0) {
+ return false;
+ }
+ // Reject trailing content after the first value, e.g. `1 2` or `{} []`,
so only a single
+ // well-formed JSON value is accepted.
+ return jsonParser.nextToken() == null;
Review Comment:
**Nit (P3):** Any trailing token already decides the answer, but this call
runs inside the outer `try` whose `StreamConstraintsException` handler raises
`IS_JSON_INPUT_LIMIT_EXCEEDED`. Suppose a complete value is followed by a
number longer than 1000 characters, such as `concat('{} ', repeat('9', 1001))`
under `IS JSON OBJECT` or `concat('1 ', repeat('9', 1001))` under `IS JSON`.
Jackson rejects that trailing number while producing its token, the same eager
check the first-token handler relies on. The query then fails with
`NUMBER_LENGTH` instead of returning false (true for `IS NOT JSON`).
That contradicts the Javadoc and `sql-ref-is-json.md`, which say trailing
content returns false and the error applies only when checking the input
requires going past a limit. It affects every shape whose first token matches,
so one such row aborts a `WHERE payload IS JSON` filter. Giving this call its
own handler that returns false would fix it. No test covers trailing over-limit
content yet.
This extends an [existing
discussion](https://github.com/apache/spark/pull/58024#issuecomment-6587687004).
**Verification:**
- **Regression:** For every shape whose first token matches (no qualifier or
VALUE, OBJECT, ARRAY and SCALAR), a complete value followed by whitespace and
an integer or floating-point number longer than the maximum number length
returns false, and the IS NOT JSON form returns true, under interpreted and
generated evaluation. Examples: concat('{} ', repeat('9', 1001)) IS JSON
OBJECT, concat('[1] ', repeat('9', 1001)) IS JSON ARRAY, concat('1 ',
repeat('9', 1001)) IS JSON and concat('"a" 0.', repeat('1', 1001)) IS JSON
SCALAR.
- **Behavior:** Existing limit cases still raise NESTING_DEPTH,
NUMBER_LENGTH and NAME_LENGTH with the same parameters, and trailing content
such as '1 2' and '{} []' still returns false.
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala:
##########
@@ -291,6 +291,45 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("a is not distinct from b", $"a" <=> $"b")
}
+ test("SPARK-58793: is json expressions") {
+ assertEqual("a is json", IsJson($"a", IsJsonShape.Any))
+ assertEqual("a is not json", !IsJson($"a", IsJsonShape.Any))
+ assertEqual("a is json value", IsJson($"a", IsJsonShape.Value))
+ assertEqual("a is json object", IsJson($"a", IsJsonShape.Object))
+ assertEqual("a is not json array", !IsJson($"a", IsJsonShape.Array))
+ assertEqual("a is json scalar", IsJson($"a", IsJsonShape.Scalar))
+ // A non-type ARRAY/MAP/STRUCT keyword must not leave the lexer in
complex-type mode, which
+ // would make a later `>>` or `>>>` lex as two `>` tokens and fail to
parse.
+ assertEqual("(a is json array) = (b >> 1)",
+ IsJson($"a", IsJsonShape.Array) === ShiftRight($"b", 1))
+ assertEqual("(a is json array) = (b >>> 1)",
+ IsJson($"a", IsJsonShape.Array) === ShiftRightUnsigned($"b", 1))
+ assertEqual("array(a) = array(b >> 1)",
+ $"array".function($"a") === $"array".function(ShiftRight($"b", 1)))
+ assertEqual("map(a, b >> 1)", $"map".function($"a", ShiftRight($"b", 1)))
+ assertEqual("struct(a, b >>> 1)", CreateStruct(Seq($"a",
ShiftRightUnsigned($"b", 1))))
+ // Angle-bracketed complex types still split consecutive closing brackets,
including when
+ // whitespace or comments separate the keyword from '<'.
+ assertEqual("cast(a as array<array<int>>) = b >> 1",
+ Cast($"a", ArrayType(ArrayType(IntegerType))) === ShiftRight($"b", 1))
+ // Each comment form precedes every type keyword, so the closers only
split if skipping that
Review Comment:
**Nit (P3):** The inputs below put each comment form after every type
keyword, between the keyword and its `<` (`array $c<array $c<int>>`). That is
the gap `isFollowedByLessThan` has to skip. A comment placed before a keyword
would not exercise the lookahead at all, so this description is inverted.
Someone who follows it could move the comments and silently lose the coverage
the loop exists to provide. Saying "after every type keyword, before its `<`"
would match the inputs.
**Verification:**
- **Inspection:** The comment above the comment-form loop describes each
comment as following a type keyword and preceding its '<', matching the inputs.
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala:
##########
@@ -291,6 +291,45 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("a is not distinct from b", $"a" <=> $"b")
}
+ test("SPARK-58793: is json expressions") {
+ assertEqual("a is json", IsJson($"a", IsJsonShape.Any))
+ assertEqual("a is not json", !IsJson($"a", IsJsonShape.Any))
+ assertEqual("a is json value", IsJson($"a", IsJsonShape.Value))
+ assertEqual("a is json object", IsJson($"a", IsJsonShape.Object))
+ assertEqual("a is not json array", !IsJson($"a", IsJsonShape.Array))
+ assertEqual("a is json scalar", IsJson($"a", IsJsonShape.Scalar))
+ // A non-type ARRAY/MAP/STRUCT keyword must not leave the lexer in
complex-type mode, which
+ // would make a later `>>` or `>>>` lex as two `>` tokens and fail to
parse.
+ assertEqual("(a is json array) = (b >> 1)",
+ IsJson($"a", IsJsonShape.Array) === ShiftRight($"b", 1))
+ assertEqual("(a is json array) = (b >>> 1)",
+ IsJson($"a", IsJsonShape.Array) === ShiftRightUnsigned($"b", 1))
+ assertEqual("array(a) = array(b >> 1)",
+ $"array".function($"a") === $"array".function(ShiftRight($"b", 1)))
+ assertEqual("map(a, b >> 1)", $"map".function($"a", ShiftRight($"b", 1)))
+ assertEqual("struct(a, b >>> 1)", CreateStruct(Seq($"a",
ShiftRightUnsigned($"b", 1))))
+ // Angle-bracketed complex types still split consecutive closing brackets,
including when
+ // whitespace or comments separate the keyword from '<'.
+ assertEqual("cast(a as array<array<int>>) = b >> 1",
+ Cast($"a", ArrayType(ArrayType(IntegerType))) === ShiftRight($"b", 1))
+ // Each comment form precedes every type keyword, so the closers only
split if skipping that
+ // comment form works: a single counted keyword would be enough to keep
`>>` apart.
+ Seq("/* c */", "/* a /* b */ c */", "-- c\n", "-- c\r\n", "-- c\r", "-- c
\\\n c\n")
Review Comment:
**Nit (P3):** These cases separate each keyword from `<` with an ASCII space
and a comment. `isFollowedByLessThan` also has to skip the WS rule's no-break
spaces (U+00A0, U+2007 and U+202F), and only its `Character.isSpaceChar` check
does that. If that disjunct were dropped, or the check narrowed to
`Character.isWhitespace`, a type such as `cast(a as
array\u00A0<array\u00A0<int>>)` would lex its closers as a shift and fail to
parse, although it parses on master, and every test would stay green. Adding
the three no-break spaces, written as Unicode escapes, to these two- and
three-level cases would pin the lexer comment's claim that the check covers
every character of the WS rule.
**Verification:**
- **Regression:** The new cases fail if the separator check is narrowed to
Character.isWhitespace, because the closers would then lex as a shift operator.
##########
sql/core/src/test/resources/sql-tests/inputs/json-functions.sql:
##########
@@ -335,3 +335,133 @@ select json_object('a' VALUE '{"b":1}');
select json_object('a' VALUE 1 RETURNING INT);
-- JSON_OBJECT error: malformed FORMAT JSON value
select json_object('a' VALUE '{bad' FORMAT JSON);
+
+-- IS [NOT] JSON
+-- basic well-formedness
+select '{"a":1}' IS JSON;
+select '[1,2]' IS JSON;
+select '42' IS JSON;
+select '"hello"' IS JSON;
+select 'not json' IS JSON;
+select '' IS JSON;
+-- IS NOT JSON
+select '{"a":1}' IS NOT JSON;
+select 'bad' IS NOT JSON;
+-- NULL propagates
+select cast(null as string) IS JSON;
+select cast(null as string) IS NOT JSON;
+-- shape qualifiers
+select '{"a":1}' IS JSON OBJECT;
+select '[1,2]' IS JSON OBJECT;
+select '[1,2]' IS JSON ARRAY;
+select '{"a":1}' IS JSON ARRAY;
+-- IS JSON VALUE: any valid JSON value (including objects and arrays)
+select '42' IS JSON VALUE;
+select '"hello"' IS JSON VALUE;
+select '{"a":1}' IS JSON VALUE;
+select '[1,2]' IS JSON VALUE;
+select 'bad' IS JSON VALUE;
+-- IS JSON SCALAR: only primitives (string, number, boolean, null -- not
object/array)
+select '42' IS JSON SCALAR;
+select '"hello"' IS JSON SCALAR;
+select 'true' IS JSON SCALAR;
+select 'false' IS JSON SCALAR;
+select '[1]' IS JSON SCALAR;
+select '{"a":1}' IS JSON SCALAR;
+-- IS NOT JSON with shape qualifiers
+select '{"a":1}' IS NOT JSON OBJECT;
+select '[1,2]' IS NOT JSON ARRAY;
+select '{"a":1}' IS NOT JSON VALUE;
+select 'bad' IS NOT JSON VALUE;
+select '{"a":1}' IS NOT JSON SCALAR;
+select '42' IS NOT JSON SCALAR;
+-- JSON null literal is valid JSON and counts as a scalar
+select 'null' IS JSON;
+select 'null' IS JSON SCALAR;
+select 'null' IS JSON VALUE;
+select 'null' IS JSON OBJECT;
+-- strict ANSI parsing: Hive-style leniency (single quotes) is rejected
+select '{''a'':1}' IS JSON;
+select '[''a'', ''b'']' IS JSON;
+-- trailing content after a well-formed value is rejected
+select '1 2' IS JSON;
+select '{} []' IS JSON;
+-- other parser extensions are rejected: comments, unquoted names,
NaN/Infinity, leading zeros
+select '/* c */ {}' IS JSON, '{a:1}' IS JSON, 'NaN' IS JSON, 'Infinity' IS
JSON, '01' IS JSON;
+-- whitespace-only input is not a JSON value
+select ' ' IS JSON;
+-- duplicate field names are accepted
+select '{"a":1,"a":2}' IS JSON OBJECT;
+-- input beyond the parser limits raises an error, like parse_json
+select concat(repeat('[', 1001), repeat(']', 1001)) IS JSON;
+-- a shape mismatch is decided by the first token even for over-limit input:
an over-long root
+-- number is neither an object nor an array, so those shapes return false
instead of raising
+select repeat('9', 1001) IS JSON OBJECT, repeat('9', 1001) IS JSON ARRAY;
+-- structurally malformed containers are rejected: truncated input, trailing
commas, missing
+-- values, and a mismatched closer
+select '{' IS JSON, '[1,' IS JSON, '{"a"' IS JSON;
+select '[1,]' IS JSON ARRAY, '{"a":1,}' IS JSON OBJECT, '[1,,2]' IS JSON
ARRAY, '[1}' IS JSON ARRAY;
+-- input that is not valid UTF-8 is rejected (lone continuation byte, encoded
surrogate)
+select cast(X'228022' AS STRING) IS JSON;
+select cast(X'22EDA08022' AS STRING) IS JSON;
+-- unpaired surrogate escapes are rejected in values and field names, also
when nested
+select '"\\uD835"' IS JSON;
+select '"\\uDC00"' IS JSON SCALAR;
+select '{"\\uD835":1}' IS JSON OBJECT;
+select '[{"a":["\\uDC00"]}]' IS JSON ARRAY;
+select '"\\uD835\\uDC00"' IS JSON;
+-- shift operators after the ARRAY qualifier
+select '[1]' IS JSON ARRAY, 8 >> 1, 8 >>> 1;
+-- collated and trim-collated operands behave like STRING
+select '{"a":1}' COLLATE UTF8_LCASE IS JSON OBJECT, '[1] ' COLLATE
UTF8_LCASE_RTRIM IS JSON ARRAY;
+select 'bad' COLLATE UNICODE IS JSON;
+-- by default a CHAR/VARCHAR cast target is rewritten to STRING at parse time,
so this case
+-- exercises a STRING operand
+select cast('[1]' AS VARCHAR(10)) IS JSON ARRAY;
+-- with first-class CHAR/VARCHAR types (preserveCharVarcharTypeInfo or
standard semantics), the
+-- cast keeps its type, so IS JSON gets a real VARCHAR or CHAR operand
Review Comment:
**Nit (P3):** This is true under `preserveCharVarcharTypeInfo` but not under
standard semantics. There, type coercion promotes the operand to STRING, as the
analyzer golden records: `is_json(cast(cast([1] as varchar(10)) as string),
Array)`. The standard-semantics block also has only VARCHAR casts, so CHAR
promotion is not exercised there. The comment could describe each config
separately, and a CHAR case in the standard-semantics block, with the golden
files regenerated, would close the gap.
**Verification:**
- **Behavior:** Under standard semantics, cast('[1]' AS CHAR(10)) IS JSON
ARRAY returns true, and its analyzed plan shows the operand promoted to STRING.
- **Inspection:** The comment describes real VARCHAR and CHAR operands only
under preserveCharVarcharTypeInfo, and STRING promotion under standard
semantics.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
