This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new 5f396538bfc [SPARK-38990][SQL] Avoid `NullPointerException` when evaluating date_trunc/trunc format as a bound reference 5f396538bfc is described below commit 5f396538bfcba7de7ab12dfa7621e7976f63fa92 Author: Bruce Robbins <bersprock...@gmail.com> AuthorDate: Fri Apr 22 12:30:34 2022 +0900 [SPARK-38990][SQL] Avoid `NullPointerException` when evaluating date_trunc/trunc format as a bound reference ### What changes were proposed in this pull request? Change `TruncInstant.evalHelper` to pass the input row to `format.eval` when `format` is a not a literal (and therefore might be a bound reference). ### Why are the changes needed? This query fails with a `java.lang.NullPointerException`: ``` select date_trunc(col1, col2) from values ('week', timestamp'2012-01-01') as data(col1, col2); ``` This only happens if the data comes from an inline table. When the source is an inline table, `ConvertToLocalRelation` attempts to evaluate the function against the data in interpreted mode. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Update to unit tests. Closes #36312 from bersprockets/date_trunc_issue. Authored-by: Bruce Robbins <bersprock...@gmail.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> (cherry picked from commit 2e4f4abf553cedec1fa8611b9494a01d24e6238a) Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../catalyst/expressions/datetimeExpressions.scala | 2 +- .../catalyst/expressions/DateExpressionsSuite.scala | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index ff3d898942c..17241f47e03 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -2114,7 +2114,7 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes { val level = if (format.foldable) { truncLevel } else { - DateTimeUtils.parseTruncLevel(format.eval().asInstanceOf[UTF8String]) + DateTimeUtils.parseTruncLevel(format.eval(input).asInstanceOf[UTF8String]) } if (level < minLevel) { // unknown format or too small level diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index c5d559c4501..8179186d9d1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -30,7 +30,7 @@ import scala.reflect.ClassTag import scala.util.Random import org.apache.spark.{SparkFunSuite, SparkUpgradeException} -import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils, TimestampFormatter} import org.apache.spark.sql.catalyst.util.DateTimeConstants._ @@ -754,6 +754,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation( TruncDate(Literal.create(input, DateType), NonFoldableLiteral.create(fmt, StringType)), expected) + // SPARK-38990: ensure that evaluation with input rows also works + val catalystInput = CatalystTypeConverters.convertToCatalyst(input) + val inputRow = InternalRow(catalystInput, UTF8String.fromString(fmt)) + checkEvaluation( + TruncDate( + BoundReference(ordinal = 0, dataType = DateType, nullable = true), + BoundReference(ordinal = 1, dataType = StringType, nullable = true)), + expected, + inputRow) } test("TruncDate") { @@ -780,6 +789,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { TruncTimestamp( NonFoldableLiteral.create(fmt, StringType), Literal.create(input, TimestampType)), expected) + // SPARK-38990: ensure that evaluation with input rows also works + val catalystInput = CatalystTypeConverters.convertToCatalyst(input) + val inputRow = InternalRow(UTF8String.fromString(fmt), catalystInput) + checkEvaluation( + TruncTimestamp( + BoundReference(ordinal = 0, dataType = StringType, nullable = true), + BoundReference(ordinal = 1, dataType = TimestampType, nullable = true)), + expected, + inputRow) } test("TruncTimestamp") { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org