This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new b3a40580ddec [SPARK-57880][SQL] Reject out-of-INT32 `variant_get`
array indices
b3a40580ddec is described below
commit b3a40580ddec12f48edb328110b2e76488d379e5
Author: bojana-db <[email protected]>
AuthorDate: Fri Jul 3 00:12:39 2026 +0800
[SPARK-57880][SQL] Reject out-of-INT32 `variant_get` array indices
### What changes were proposed in this pull request?
The variant path parser accepts an array subscript $[N], but neither engine
validated that N fits in a signed 32-bit int. `VariantPathParser` called
`index.toInt`, which threw a raw `NumberFormatException` (surfacing as an
internal error) for indices above `Int.MaxValue`.
### Why are the changes needed?
Java internal error should not be visible to customers.
### Does this PR introduce _any_ user-facing change?
Only changing error output for edge case.
### How was this patch tested?
Unit tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #56960 from bojana-db/overflow-fix.
Authored-by: bojana-db <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit e3e6d9eb31012244692c043eb92a582ae3c3751b)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../expressions/variant/variantExpressions.scala | 10 ++++++---
.../variant/VariantExpressionSuite.scala | 25 +++++++++++++++++-----
2 files changed, 27 insertions(+), 8 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index bf5e4183de47..e0ac976e5100 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -218,9 +218,13 @@ object VariantPathParser extends RegexParsers {
private val parser: Parser[List[VariantPathSegment]] = phrase(root ~>
rep(key | index))
def parse(str: String): Option[Array[VariantPathSegment]] = {
- this.parseAll(parser, str) match {
- case Success(result, _) => Some(result.toArray)
- case _ => None
+ try {
+ this.parseAll(parser, str) match {
+ case Success(result, _) => Some(result.toArray)
+ case _ => None
+ }
+ } catch {
+ case _: NumberFormatException => None
}
}
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index 467cb6335a2e..73fdc48fa585 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -668,11 +668,15 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
test("variant_get path") {
def checkInvalidPath(path: String): Unit = {
- checkErrorInExpression[SparkRuntimeException](
- variantGet("0", path, IntegerType),
- "INVALID_VARIANT_GET_PATH",
- Map("path" -> path, "functionName" -> "`variant_get`")
- )
+ for ((expr, fn) <- Seq(
+ variantGet("0", path, IntegerType) -> "`variant_get`",
+ tryVariantGet("0", path, IntegerType) -> "`try_variant_get`")) {
+ checkErrorInExpression[SparkRuntimeException](
+ expr,
+ "INVALID_VARIANT_GET_PATH",
+ Map("path" -> path, "functionName" -> fn)
+ )
+ }
}
testVariantGet("""{"1": {"2": {"3": [4]}}}""", "$.1.2.3[0]", IntegerType,
4)
@@ -680,11 +684,14 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
// scalastyle:off nonascii
testVariantGet("""{"你好": {"世界": "hello"}}""", """$['你好']["世界"]""",
StringType, "hello")
// scalastyle:on nonascii
+ testVariantGet("[1, 2, 3]", "$[2147483647]", IntegerType, null)
checkInvalidPath("")
checkInvalidPath(".a")
checkInvalidPath("$1")
checkInvalidPath("$[-1]")
+ checkInvalidPath("$[2147483648]")
+ checkInvalidPath("$[4294967296]")
checkInvalidPath("""$['"]""")
checkInvalidPath("$[\"\"\"]")
@@ -1306,6 +1313,14 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
"INVALID_VARIANT_PATH",
Map("path" -> ".a", "functionName" -> "`variant_delete`"))
+ for (path <- Seq("$[2147483648]", "$[4294967296]")) {
+ checkErrorInExpression[SparkRuntimeException](
+ ResolveTimeZone.resolveTimeZones(
+ VariantDelete(Seq(Literal(parseJson("[1, 2, 3]")), Literal(path)))),
+ "INVALID_VARIANT_PATH",
+ Map("path" -> path, "functionName" -> "`variant_delete`"))
+ }
+
val noPaths = VariantDelete(Seq(Literal(parseJson("""{"a": 1}"""))))
intercept[org.apache.spark.sql.AnalysisException] {
noPaths.checkInputDataTypes()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
